From 9f273171cd26142fc2f3930d0d54907957d97267 Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:40:39 +0100 Subject: [PATCH 01/17] Harden deployment readiness and validation environment cleanup (#5119) --- .devcontainer/Dockerfile | 2 +- .github/linters/actionlint.yml | 3 + .../scripts/ci_environment_workflow.test.js | 83 +++- .github/scripts/package-lock.json | 4 +- .github/scripts/package.json | 2 +- .../workflows/build_validation_develop.yml | 20 + .github/workflows/deploy_tre_branch.yml | 10 +- .github/workflows/deploy_tre_reusable.yml | 45 ++- CHANGELOG.md | 1 + config_schema.json | 5 +- core/terraform/deploy.sh | 5 +- core/terraform/json-to-env.sh | 50 ++- .../mgmt_storage_private_endpoint.sh | 134 +++++++ core/terraform/outputs.sh | 88 ++++- core/version.txt | 2 +- devops/scripts/ci_cleanup_activity.py | 127 ++++++ devops/scripts/ci_cleanup_scope.py | 19 +- devops/scripts/clean_ci_validation_envs.sh | 24 +- devops/scripts/recover_bootstrap_lease.py | 1 + devops/scripts/terraform_init.sh | 11 +- devops/scripts/verify_deployment_lock.py | 81 ++++ devops/tests/test_bootstrap.py | 75 ++++ devops/tests/test_ci_cleanup_activity.py | 178 +++++++++ devops/tests/test_ci_cleanup_scope.py | 38 +- devops/tests/test_clean_ci_validation_envs.py | 40 ++ devops/tests/test_core_outputs.py | 367 ++++++++++++++++++ .../test_mgmt_storage_private_endpoint.py | 262 +++++++++++++ devops/tests/test_validate_ci_cleanup.py | 2 + devops/tests/test_verify_deployment_lock.py | 180 +++++++++ devops/version.txt | 2 +- 30 files changed, 1799 insertions(+), 62 deletions(-) create mode 100755 core/terraform/mgmt_storage_private_endpoint.sh mode change 100644 => 100755 core/terraform/outputs.sh create mode 100644 devops/scripts/ci_cleanup_activity.py create mode 100644 devops/scripts/verify_deployment_lock.py create mode 100644 devops/tests/test_ci_cleanup_activity.py create mode 100644 devops/tests/test_core_outputs.py create mode 100644 devops/tests/test_mgmt_storage_private_endpoint.py create mode 100644 devops/tests/test_verify_deployment_lock.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a970d25859..f39a3b5958 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -43,7 +43,7 @@ ARG DOCKER_CE_VERSION="5:29.1.3-1~debian.12~bookworm" ARG DOCKER_CE_CLI_VERSION="5:29.1.3-1~debian.12~bookworm" ARG DOCKER_CONTAINERD_VERSION="2.2.1-1~debian.12~bookworm" ARG DOCKER_COMPOSE_PLUGIN_VERSION="5.0.0-1~debian.12~bookworm" -RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release --no-install-recommends \ +RUN apt-get update && apt-get install -y ca-certificates curl dnsutils gnupg lsb-release --no-install-recommends \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ | tee /etc/apt/sources.list.d/docker.list > /dev/null \ diff --git a/.github/linters/actionlint.yml b/.github/linters/actionlint.yml index 7a0c2e52f8..daba70a4fd 100644 --- a/.github/linters/actionlint.yml +++ b/.github/linters/actionlint.yml @@ -4,6 +4,9 @@ # Remove these exact parser exceptions when actionlint supports queue. # ci_environment_workflow.test.js validates all reference group settings. paths: + .github/workflows/deploy_tre_branch.yml: + ignore: + - '^unexpected key "queue" for "concurrency" section\. expected one of "cancel-in-progress", "group"$' .github/workflows/clean_validation_envs.yml: ignore: - '^unexpected key "queue" for "concurrency" section\. expected one of "cancel-in-progress", "group"$' diff --git a/.github/scripts/ci_environment_workflow.test.js b/.github/scripts/ci_environment_workflow.test.js index 6500a51472..14cf616082 100644 --- a/.github/scripts/ci_environment_workflow.test.js +++ b/.github/scripts/ci_environment_workflow.test.js @@ -3,6 +3,7 @@ const os = require('os'); const path = require('path'); const { execFileSync } = require('child_process'); const yaml = require('js-yaml'); +const vm = require('vm'); const root = path.resolve(__dirname, '../..'); const workflow = name => yaml.load(fs.readFileSync(path.join(root, '.github/workflows', name), 'utf8')); @@ -10,7 +11,6 @@ const workflow = name => yaml.load(fs.readFileSync(path.join(root, '.github/work describe('reference concurrency queues', () => { test.each([ ['deploy_tre.yml', 'run-deploy-tre-main', '${{ github.ref }}'], - ['deploy_tre_branch.yml', 'run-deploy-tre-not-main', '${{ github.ref }}'], ['pr_comment_bot.yml', 'run_test', '${{ needs.pr_comment.outputs.ciGitRef }}'] ])('%s routes deployment into the shared queue without an outer queue', (file, job, ref) => { const caller = workflow(file); @@ -18,6 +18,7 @@ describe('reference concurrency queues', () => { expect(caller.jobs[job].concurrency).toBeUndefined(); expect(caller.jobs[job].uses).toBe('./.github/workflows/deploy_tre_reusable.yml'); expect(caller.jobs[job].with.ciGitRef).toBe(ref); + expect(caller.jobs[job].with.callerOwnsDeploymentLock).toBeUndefined(); // Preparation may run independently, but must not add another waiting group. const needs = caller.jobs[job].needs || []; for (const dependency of Array.isArray(needs) ? needs : [needs]) { @@ -25,8 +26,30 @@ describe('reference concurrency queues', () => { } }); + test('branch validation owns the reference queue before preparation', () => { + const caller = workflow('deploy_tre_branch.yml'); + expect(caller.concurrency).toEqual({ group: 'deploy-${{ github.ref }}', 'cancel-in-progress': false, queue: 'max' }); + const job = caller.jobs['run-deploy-tre-not-main']; + expect(job.uses).toBe('./.github/workflows/deploy_tre_reusable.yml'); + expect(job.with.ciGitRef).toBe('${{ github.ref }}'); + expect(job.with.callerOwnsDeploymentLock).toBe(true); + expect(Object.values(caller.jobs).every(value => value.concurrency === undefined)).toBe(true); + }); + + test('delegation avoids a nested reference lock and preserves the default queue', () => { + const reusable = workflow('deploy_tre_reusable.yml'); + expect(reusable.on.workflow_call.inputs.callerOwnsDeploymentLock).toMatchObject({ + type: 'boolean', required: false, default: false + }); + expect(reusable.concurrency).toEqual({ + group: "${{ inputs.callerOwnsDeploymentLock && format('deploy-delegated-{0}-{1}', github.run_id, github.run_attempt) " + + "|| format('deploy-{0}', inputs.ciGitRef) }}", + 'cancel-in-progress': false, + queue: 'max' + }); + }); + test.each([ - ['deploy_tre_reusable.yml', null, 'deploy-${{ inputs.ciGitRef }}'], ['pr_comment_bot.yml', 'destroy_pr_env', 'deploy-${{ needs.pr_comment.outputs.ciGitRef }}'], ['pr_comment_bot.yml', 'destroy_branch_env', 'deploy-${{ needs.pr_comment.outputs.branchCiGitRef }}'], ['clean_validation_envs.yml', 'clean', 'deploy-${{ matrix.ref }}'], @@ -53,6 +76,62 @@ describe('reference concurrency queues', () => { }); }); +function e2eCondition(job, lockResult, skipDeployment) { + const needs = Object.fromEntries(job.needs.map(name => [name, { + result: name === 'verify_deployment_lock' ? lockResult + : name.startsWith('e2e_tests_') ? 'success' : (skipDeployment ? 'skipped' : 'success') + }])); + const inputs = { e2eBundle: '', e2eTestsCustomSelector: 'airlock', skipDeployment }; + const results = Object.values(needs).map(value => value.result); + const expression = job.if.trim().slice(3, -2).replaceAll('needs.*.result', 'results'); + return vm.runInNewContext(expression, { + needs, inputs, results, always: () => true, contains: (values, value) => values.includes(value) + }, { timeout: 100 }); +} + +describe('delegated deployment guard', () => { + test('the guard reads workflow source and has no Azure credentials', () => { + const job = workflow('deploy_tre_reusable.yml').jobs.verify_deployment_lock; + expect(job.permissions).toEqual({ actions: 'read', contents: 'read' }); + expect(job.steps[0].if).toBe('inputs.callerOwnsDeploymentLock'); + expect(job.steps[0].with).toEqual({ 'persist-credentials': false, ref: '${{ github.sha }}' }); + const check = job.steps.find(step => step.run?.includes('verify_deployment_lock.py')); + expect(check.if).toBe('inputs.callerOwnsDeploymentLock'); + expect(check.env).toEqual({ + CI_DEPLOYMENT_LOCK_DELEGATED: '${{ inputs.callerOwnsDeploymentLock }}', + CI_GIT_REF: '${{ inputs.ciGitRef }}', + GITHUB_TOKEN: '${{ github.token }}' + }); + const fallback = job.steps.find(step => step.if === '${{ !inputs.callerOwnsDeploymentLock }}'); + expect(fallback.run).toBe('echo "The reusable workflow holds the deployment reference lock."'); + }); + + test('every Azure job depends on the guard', () => { + const jobs = workflow('deploy_tre_reusable.yml').jobs; + expect(Object.keys(jobs).filter(name => !jobs[name].needs)).toEqual(['verify_deployment_lock']); + expect(jobs.deploy_management.needs).toEqual(['verify_deployment_lock']); + for (const name of ['e2e_tests_smoke', 'e2e_tests_custom']) { + expect(jobs[name].needs).toContain('verify_deployment_lock'); + } + }); + + test.each(['success', 'failure', 'cancelled', 'skipped'])('E2E requires guard success when its result is %s', result => { + const jobs = workflow('deploy_tre_reusable.yml').jobs; + for (const name of ['e2e_tests_smoke', 'e2e_tests_custom']) { + for (const skipDeployment of [true, false]) { + expect(e2eCondition(jobs[name], result, skipDeployment)).toBe(result === 'success'); + } + } + }); + + test('guard changes select the DevOps tests', () => { + const filter = workflow('build_validation_develop.yml').jobs; + const steps = Object.values(filter).flatMap(job => job.steps || []); + const paths = yaml.load(steps.find(step => step.id === 'filter').with.filters); + expect(paths.ci_cleanup).toContain('devops/scripts/verify_deployment_lock.py'); + }); +}); + function prepare(job, location, ref, output) { const step = job.steps.find(value => value.id === 'environment-id' || value.id === 'run-id'); execFileSync('bash', ['-c', step.run], { diff --git a/.github/scripts/package-lock.json b/.github/scripts/package-lock.json index 11613c9df6..b28d9ccc76 100644 --- a/.github/scripts/package-lock.json +++ b/.github/scripts/package-lock.json @@ -1,12 +1,12 @@ { "name": "build-scripts", - "version": "1.0.3", + "version": "1.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "build-scripts", - "version": "1.0.3", + "version": "1.0.4", "license": "MIT", "devDependencies": { "jest": "^30.2.0", diff --git a/.github/scripts/package.json b/.github/scripts/package.json index 58e755bb06..0452315bc7 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -1,6 +1,6 @@ { "name": "build-scripts", - "version": "1.0.3", + "version": "1.0.4", "description": "Scripts used in the GitHub workflows", "main": "build.js", "license": "MIT", diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index 956c166464..722c848a54 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -65,6 +65,12 @@ jobs: - '**/porter-build-context.env' ci_cleanup: - 'Makefile' + - 'config_schema.json' + - 'core/terraform/outputs.tf' + - 'core/terraform/outputs.sh' + - 'core/terraform/json-to-env.sh' + - 'devops/scripts/load_env.sh' + - 'devops/scripts/load_and_validate_env.sh' - 'core/terraform/scripts/letsencrypt.sh' - 'core/terraform/scripts/validate_certbot.sh' - 'devops/scripts/clean_ci_validation_envs.sh' @@ -74,10 +80,14 @@ jobs: - 'devops/scripts/recover_bootstrap_lease.py' - 'devops/scripts/ci_environment_id.py' - 'devops/scripts/ci_cleanup_scope.py' + - 'devops/scripts/ci_cleanup_activity.py' + - 'devops/scripts/verify_deployment_lock.py' - 'devops/scripts/destroy_ci_ref_env.py' - 'devops/scripts/destroy_env_no_terraform.sh' - 'devops/scripts/validate_ci_cleanup.py' - 'devops/scripts/kv_add_network_exception.sh' + - 'core/terraform/deploy.sh' + - 'core/terraform/mgmt_storage_private_endpoint.sh' - 'devops/terraform/**' - '.github/actions/devcontainer_run_command/action.yml' - '.github/workflows/deploy_tre_reusable.yml' @@ -129,6 +139,16 @@ jobs: if: ${{ steps.filter.outputs.ci_cleanup == 'true' }} run: python3 -m unittest discover -s devops/tests -v + - name: Test core outputs in the development-container base + if: ${{ steps.filter.outputs.ci_cleanup == 'true' }} + # Bookworm provides jq 1.6; the hosted runner has a newer jq version. + run: | + docker run --rm --network none \ + --mount "type=bind,source=${PWD},target=/workspaces/tre,readonly" \ + --workdir /workspaces/tre --entrypoint /bin/bash \ + mcr.microsoft.com/vscode/devcontainers/python:dev-3.12-bookworm \ + -c 'jq --version; python3 -m unittest discover -s devops/tests -p test_core_outputs.py -v' + - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 if: ${{ steps.filter.outputs.terraform == 'true' }} with: diff --git a/.github/workflows/deploy_tre_branch.yml b/.github/workflows/deploy_tre_branch.yml index c37788e1ae..cd1d01cb5c 100644 --- a/.github/workflows/deploy_tre_branch.yml +++ b/.github/workflows/deploy_tre_branch.yml @@ -35,8 +35,13 @@ on: # yamllint disable-line rule:truthy default: false required: false -# The reusable deployment workflow holds the shared reference queue. -# Separate caller queues can discard deployments before they reach that group. +# Register branch validation before Preparation can start. The reusable workflow +# delegates this reference lock to the caller. Older workflow reruns keep their +# original locking behaviour; dispatch this workflow to use the early lock. +concurrency: + group: deploy-${{ github.ref }} + cancel-in-progress: false + queue: max permissions: contents: read @@ -90,6 +95,7 @@ jobs: actions: read # Required by the reusable management job with: ciGitRef: ${{ github.ref }} + callerOwnsDeploymentLock: true ciEnvironmentId: ${{ needs.prepare-not-main.outputs.refid }} prHeadSha: ${{ github.sha }} e2eTestsCustomSelector: ${{ github.event.inputs.e2eTestsCustomSelector }} diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 1a794f107f..475da27073 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -18,6 +18,11 @@ on: # yamllint disable-line rule:truthy description: The git ref to use in annotations to associate a deployment with the code that triggered it type: string required: true + callerOwnsDeploymentLock: + description: The branch caller holds the deployment reference lock for its entire workflow + type: boolean + required: false + default: false ciEnvironmentId: description: Isolated CI environment ID derived from the Git reference, Azure cloud and location type: string @@ -128,10 +133,12 @@ on: # yamllint disable-line rule:truthy description: "" required: false -# This will prevent multiple runs of this entire workflow. -# We should NOT cancel in progress runs as that can destabilize the environment. +# Other callers retain the reference lock here. A branch caller already holds +# it, so use a run-specific inner group and verify the outer owner before work. concurrency: - group: deploy-${{ inputs.ciGitRef }} + group: >- + ${{ inputs.callerOwnsDeploymentLock && format('deploy-delegated-{0}-{1}', github.run_id, github.run_attempt) + || format('deploy-{0}', inputs.ciGitRef) }} cancel-in-progress: false queue: max @@ -139,8 +146,34 @@ env: CI_ENVIRONMENT_ID: ${{ inputs.ciEnvironmentId }} jobs: + verify_deployment_lock: + name: Verify deployment lock + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: read + contents: read + steps: + - name: Checkout workflow source + if: inputs.callerOwnsDeploymentLock + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + ref: ${{ github.sha }} + - name: Verify the delegated reference lock + if: inputs.callerOwnsDeploymentLock + env: + CI_DEPLOYMENT_LOCK_DELEGATED: ${{ inputs.callerOwnsDeploymentLock }} + CI_GIT_REF: ${{ inputs.ciGitRef }} + GITHUB_TOKEN: ${{ github.token }} + run: python3 devops/scripts/verify_deployment_lock.py + - name: Use the reusable workflow reference lock + if: ${{ !inputs.callerOwnsDeploymentLock }} + run: echo "The reusable workflow holds the deployment reference lock." + deploy_management: name: Deploy Management + needs: [verify_deployment_lock] if: ${{ !inputs.skipDeployment }} runs-on: ubuntu-latest permissions: @@ -861,12 +894,13 @@ jobs: # always() lets this job still evaluate its guard and run against the existing PR environment. if: >- ${{ always() && + needs.verify_deployment_lock.result == 'success' && (inputs.skipDeployment || !contains(needs.*.result, 'skipped')) && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} runs-on: ubuntu-latest environment: ${{ inputs.environmentName }} - needs: [deploy_shared_services, register_bundles, deploy_ui] + needs: [verify_deployment_lock, deploy_shared_services, register_bundles, deploy_ui] timeout-minutes: 10 steps: - name: Checkout @@ -911,13 +945,14 @@ jobs: # Same as smoke tests: skipDeployment intentionally bypasses deploy jobs but still runs E2E tests. if: >- ${{ always() && inputs.e2eTestsCustomSelector != '' && + needs.verify_deployment_lock.result == 'success' && (inputs.skipDeployment || !contains(needs.*.result, 'skipped')) && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} runs-on: ubuntu-latest environment: ${{ inputs.environmentName }} needs: - [deploy_shared_services, register_bundles, register_user_resource_bundles] + [verify_deployment_lock, deploy_shared_services, register_bundles, register_user_resource_bundles] timeout-minutes: 300 steps: - name: Checkout diff --git a/CHANGELOG.md b/CHANGELOG.md index b8682ce651..9690aad6e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Wait for management storage, retry recognised state-read permission failures and validate Terraform outputs. Protect active and recently used validation environments from scheduled cleanup. ([#5252](https://github.com/microsoft/AzureTRE/pull/5252)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Validate Airlock sort fields against the supported options and avoid returning database error details to callers. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Bind operation retrieval to the authorised resource and return not found for unrelated operations. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/config_schema.json b/config_schema.json index 56eba4a703..82a042aa3d 100644 --- a/config_schema.json +++ b/config_schema.json @@ -219,7 +219,10 @@ "workspace_api_client_secret": { "description": "Workspace AD Application secret. This will be created for you for future use - when creating workspaces.", "type": "string", - "minLength": 11 + "minLength": 11, + "not": { + "pattern": "['\\r\\n\\u0000]" + } } }, "required": [ diff --git a/core/terraform/deploy.sh b/core/terraform/deploy.sh index f24107f32e..a954bbf726 100755 --- a/core/terraform/deploy.sh +++ b/core/terraform/deploy.sh @@ -27,6 +27,8 @@ LOG_FILE="${TS}-tre-core.log" # core deployments. This may increase the risk of hitting Azure API rate limits. # If deployment reliability issues arise (e.g., frequent throttling), consider # lowering the parallelism value. +# The management storage private endpoint is created before the main apply because +# creating it changes the DNS of the storage account that holds the Terraform state. # This variables are loaded in for us # shellcheck disable=SC2154 ../../devops/scripts/terraform_wrapper.sh \ @@ -36,7 +38,8 @@ LOG_FILE="${TS}-tre-core.log" -n "${TF_VAR_terraform_state_container_name}" \ -k "${TRE_ID}" \ -l "${LOG_FILE}" \ - -c "terraform plan --parallelism=25 -out ${PLAN_FILE} && \ + -c "./mgmt_storage_private_endpoint.sh && \ + terraform plan --parallelism=25 -out ${PLAN_FILE} && \ terraform apply -input=false -auto-approve --parallelism=25 ${PLAN_FILE} && \ terraform output -json > ../tre_output.json" diff --git a/core/terraform/json-to-env.sh b/core/terraform/json-to-env.sh index 0360fc2cc4..a3d824b013 100755 --- a/core/terraform/json-to-env.sh +++ b/core/terraform/json-to-env.sh @@ -1,9 +1,16 @@ #!/bin/bash set -e +set -o pipefail echo "# Generated environment variables from tf output" -jq -r ' +jq -ers ' + if length != 1 or (.[0] | type) != "object" then + error("Expected one Terraform output object. Check the state and regenerate core/tre_output.json.") + else + .[0] + end + | [ { "path": "core_resource_group_name", @@ -115,25 +122,40 @@ jq -r ' "env_var": "EVENT_GRID_AIRLOCK_NOTIFICATION_TOPIC_RESOURCE_ID" }, { + # This mapping is retained for older caches but is not a root output. "path": "airlock_malware_scan_result_topic_name", - "env_var": "AIRLOCK_MALWARE_SCAN_RESULT_TOPIC_NAME" + "env_var": "AIRLOCK_MALWARE_SCAN_RESULT_TOPIC_NAME", + "optional": true } ] as $env_vars_to_extract | - with_entries( - select ( - .key as $a - | - any( $env_vars_to_extract[]; .path == $a) - ) - | - .key |= . as $old_key | ($env_vars_to_extract[] | select (.path == $old_key) | .env_var) - ) + . as $outputs | - to_entries + $env_vars_to_extract | - map("\(.key)=\"\(.value.value)\"") + map( + . as $mapping + | if $mapping.optional and ($outputs | has($mapping.path) | not) then + empty + else + $outputs[$mapping.path] + | if type != "object" then + error("Missing or invalid Terraform output: \($mapping.path). Check the state and regenerate core/tre_output.json.") + elif (.value | type) != "string" then + error("Terraform output \($mapping.path) must have a string value. Check the state and regenerate core/tre_output.json.") + elif (.value | test("\\S") | not) then + error("Terraform output \($mapping.path) is empty. Check the state and regenerate core/tre_output.json.") + # jq 1.6 treats a NUL needle as empty in contains(). + elif (.value | index("\u0000") != null or test("[\r\n\u0027]")) then + error("Terraform output \($mapping.path) contains a single quote, newline or NUL. Check the state and regenerate core/tre_output.json.") + else + # load_env.sh strips outer quotes without interpreting shell escapes. + # load_and_validate_env.sh also sources this file, so reject single quotes above. + "\($mapping.env_var)=\u0027\(.value)\u0027" + end + end + ) | .[] - ' | sed "s/\"/'/g" # replace double quote with single quote to handle special chars + ' diff --git a/core/terraform/mgmt_storage_private_endpoint.sh b/core/terraform/mgmt_storage_private_endpoint.sh new file mode 100755 index 0000000000..90a2d01707 --- /dev/null +++ b/core/terraform/mgmt_storage_private_endpoint.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# +# Reconcile the resource processor's management storage private endpoint before the main core apply. +# +# The management storage account also holds the Terraform state. Creating its private endpoint +# changes the storage account's public DNS alias chain, so state reads and writes during the same +# apply can fail with "no such host". This script reconciles the private endpoint in a targeted +# apply, then verifies the new DNS alias and backend reads before the main apply starts. +# +# Run from core/terraform after "terraform init". +# + +set -o errexit +set -o pipefail +set -o nounset + +PE_ADDRESS='module.resource_processor_vmss_porter[0].azurerm_private_endpoint.mgmtblobpe' +DNS_REQUIRED_SUCCESSES="${MGMT_STORAGE_DNS_REQUIRED_SUCCESSES:-3}" +DNS_MAX_ATTEMPTS="${MGMT_STORAGE_DNS_MAX_ATTEMPTS:-30}" +DNS_INTERVAL_SECONDS="${MGMT_STORAGE_DNS_INTERVAL_SECONDS:-10}" + +function timestamp() { + date -u +"%Y-%m-%dT%H:%M:%SZ" +} + +function state_blob_host() { + local endpoint + # shellcheck disable=SC2154 + endpoint=$(az storage account show \ + --name "${TF_VAR_mgmt_storage_account_name}" \ + --resource-group "${TF_VAR_mgmt_resource_group_name}" \ + --query primaryEndpoints.blob --output tsv) + endpoint="${endpoint#https://}" + echo "${endpoint%%/*}" +} + +function state_blob_is_reachable() { + local exists + # Check the data plane without initialising state or acquiring a state lease. + # An authenticated not-found response is sufficient for a new state blob. + # shellcheck disable=SC2154 + if ! exists=$(timeout 30 az storage blob exists \ + --account-name "${TF_VAR_mgmt_storage_account_name}" \ + --container-name "${TF_VAR_terraform_state_container_name}" \ + --name "${TRE_ID}" --auth-mode login --only-show-errors \ + --query exists --output tsv); then + return 1 + fi + case "${exists}" in + true|false) return 0 ;; + *) echo "Unexpected response from the state blob existence check" >&2; return 1 ;; + esac +} + +function wait_for_stable_dns() { + local host="$1" + local expected_alias="${host%%.*}.privatelink.${host#*.}" + local successes=0 + local attempt aliases answer backend_error + + echo "Waiting for ${expected_alias}, address resolution and backend access (${DNS_REQUIRED_SUCCESSES} consecutive checks)" + for ((attempt=1; attempt<=DNS_MAX_ATTEMPTS; attempt++)); do + aliases=$(dig +time=5 +tries=1 +short CNAME "${host}" 2>&1) || aliases="" + if ! grep -ixFq "${expected_alias}." <<< "${aliases}"; then + successes=0 + echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: expected alias not observed: ${aliases:-no answer}" + elif ! answer=$(timeout 15 getent ahosts "${host}" 2>&1) || [[ -z "${answer}" ]]; then + successes=0 + echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: ${host} did not resolve ${answer}" + elif ! backend_error=$(state_blob_is_reachable 2>&1); then + successes=0 + echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: backend read failed: ${backend_error}" + else + successes=$((successes + 1)) + echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: alias, address and backend ready (${successes}/${DNS_REQUIRED_SUCCESSES})" + echo " CNAME: ${aliases}" + echo "${answer}" | awk '{print " " $0}' + if [[ "${successes}" -ge "${DNS_REQUIRED_SUCCESSES}" ]]; then + return 0 + fi + fi + if [[ "${attempt}" -lt "${DNS_MAX_ATTEMPTS}" ]]; then + sleep "${DNS_INTERVAL_SECONDS}" + fi + done + + echo "Error: ${host} did not become ready after ${DNS_MAX_ATTEMPTS} attempts." >&2 + echo "Configured resolver diagnostics:" + dig +time=2 +tries=1 +noall +answer +comments +stats "${host}" A || true + echo "Azure resolver diagnostics (168.63.129.16):" + dig @168.63.129.16 +time=2 +tries=1 +noall +answer +comments +stats "${host}" A || true + echo "The main apply has not started. Inspect DNS and backend access, then re-run to repeat these checks." >&2 + return 1 +} + +if [[ "${TF_VAR_resource_processor_type:-vmss_porter}" != "vmss_porter" ]]; then + echo "Resource processor type is not vmss_porter; skipping management storage private endpoint step" + exit 0 +fi + +for required_command in dig getent timeout; do + if ! command -v "${required_command}" >/dev/null; then + echo "Error: ${required_command} is required. Rebuild the repository development container." >&2 + exit 1 + fi +done + +host=$(state_blob_host) +echo "$(timestamp) Planning management storage private endpoint before the main core apply" +timeout 15 getent ahosts "${host}" | awk '{print " before: " $0}' || echo " before: ${host} did not resolve" + +# State presence alone misses tainted resources, replacement and remote deletion. +# Refresh the target and its dependencies, and apply the saved plan only when needed. +PE_PLAN_FILE="$(date +"%s")-tre-core-mgmt-pe.tfplan" +if terraform plan -input=false -detailed-exitcode -target="${PE_ADDRESS}" -out "${PE_PLAN_FILE}"; then + plan_exit=0 +else + plan_exit=$? +fi +case "${plan_exit}" in + 0) + echo "Management storage private endpoint target has no planned changes; skipping targeted apply" + ;; + 2) + terraform apply -input=false -auto-approve "${PE_PLAN_FILE}" + echo "$(timestamp) Management storage private endpoint apply finished" + ;; + *) + echo "Error: management storage private endpoint plan failed (exit ${plan_exit}). The main apply has not started." >&2 + exit "${plan_exit}" + ;; +esac + +wait_for_stable_dns "${host}" diff --git a/core/terraform/outputs.sh b/core/terraform/outputs.sh old mode 100644 new mode 100755 index 4b2ffa8347..5538105986 --- a/core/terraform/outputs.sh +++ b/core/terraform/outputs.sh @@ -1,41 +1,93 @@ #!/bin/bash set -e +# This file is sourced inside Makefile && chains, where Bash ignores errexit. +# Check failures explicitly so callers cannot continue with missing outputs. # shellcheck disable=SC1091 # shellcheck disable=SC2154 source ../../devops/scripts/storage_enable_public_access.sh \ --storage-account-name "${TF_VAR_mgmt_storage_account_name}" \ - --resource-group-name "${TF_VAR_mgmt_resource_group_name}" + --resource-group-name "${TF_VAR_mgmt_resource_group_name}" || return $? -if [ ! -f ../tre_output.json ] || [ ! -s ../tre_output.json ]; then - # Connect to the remote backend of Terraform +core_outputs_file=../tre_output.json +core_outputs_temporary="" +if [ ! -s "$core_outputs_file" ]; then + # Connect to the remote backend of Terraform. export TF_LOG="" # shellcheck disable=SC2154 terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=${TRE_ID}" + -backend-config="key=${TRE_ID}" || return $? - # Convert the output to json - terraform output -json > ../tre_output.json + # Keep Terraform output temporary until it has been validated and converted. + core_outputs_temporary=$(mktemp ../tre_output.json.tmp.XXXXXX) || return $? + if terraform output -json > "$core_outputs_temporary"; then + core_outputs_file="$core_outputs_temporary" + else + core_outputs_exit=$? + rm -f "$core_outputs_temporary" + return "$core_outputs_exit" + fi fi -# Now create an .env file -./json-to-env.sh < ../tre_output.json > ../private.env +if ! jq -e 'type == "object" and length > 0' "$core_outputs_file" > /dev/null; then + if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi + echo "Terraform outputs are empty or invalid. Check the state and regenerate core/tre_output.json." >&2 + return 1 +fi -# Pull in the core templates environment variables so we can build up new key/value pairs +# Pull in the core template environment variables for the additional values. if [ -f ../.env ]; then # shellcheck disable=SC1091 - source ../.env + source ../.env || { + core_outputs_exit=$? + if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi + return "$core_outputs_exit" + } fi -# Add a few extra values to the file to help us (i.e. for local debugging api_app and resource processor) -# shellcheck disable=SC2129 -echo "TEST_WORKSPACE_APP_ID='${WORKSPACE_API_CLIENT_ID}'" >> ../private.env -echo "TEST_WORKSPACE_APP_SECRET='${WORKSPACE_API_CLIENT_SECRET}'" >> ../private.env +# Both environment readers require literal, single-line, single-quoted values. +# Bash variables cannot contain NUL bytes. +for core_env_variable in WORKSPACE_API_CLIENT_ID WORKSPACE_API_CLIENT_SECRET SUB_ID TENANT_ID; do + case "${!core_env_variable}" in + *"'"*|*$'\r'*|*$'\n'*) + if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi + echo "${core_env_variable} contains a single quote or newline. Correct the value and regenerate core/private.env." >&2 + return 1 + ;; + esac +done -# These next ones from Check Dependencies -echo "SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env -echo "AZURE_SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env -echo "AZURE_TENANT_ID='${TENANT_ID}'" >> ../private.env +# Preserve the previous environment file if conversion or writing fails. +core_env_temporary=$(mktemp ../private.env.tmp.XXXXXX) || { + core_outputs_exit=$? + if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi + return "$core_outputs_exit" +} +if ./json-to-env.sh < "$core_outputs_file" > "$core_env_temporary" && \ + printf "%s='%s'\n" \ + TEST_WORKSPACE_APP_ID "${WORKSPACE_API_CLIENT_ID}" \ + TEST_WORKSPACE_APP_SECRET "${WORKSPACE_API_CLIENT_SECRET}" \ + SUBSCRIPTION_ID "${SUB_ID}" \ + AZURE_SUBSCRIPTION_ID "${SUB_ID}" \ + AZURE_TENANT_ID "${TENANT_ID}" >> "$core_env_temporary"; then + if [ -n "$core_outputs_temporary" ]; then + mv "$core_outputs_temporary" ../tre_output.json || { + core_outputs_exit=$? + rm -f "$core_outputs_temporary" "$core_env_temporary" + return "$core_outputs_exit" + } + fi + mv "$core_env_temporary" ../private.env || { + core_outputs_exit=$? + rm -f "$core_env_temporary" + return "$core_outputs_exit" + } +else + core_outputs_exit=$? + if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi + rm -f "$core_env_temporary" + return "$core_outputs_exit" +fi diff --git a/core/version.txt b/core/version.txt index 2cfeb05209..0639f54b2f 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.18.11" +__version__ = "0.18.13" diff --git a/devops/scripts/ci_cleanup_activity.py b/devops/scripts/ci_cleanup_activity.py new file mode 100644 index 0000000000..0148b230d6 --- /dev/null +++ b/devops/scripts/ci_cleanup_activity.py @@ -0,0 +1,127 @@ +"""Defer scoped cleanup while matching validation is active or recently used a branch.""" + +from datetime import datetime, timezone +from urllib.parse import urlencode + +from recover_bootstrap_lease import RecoveryError, github, require + +ACTIVE_STATUSES = frozenset(("in_progress", "queued", "pending", "waiting", "requested")) +CLEANUP_WORKFLOW = ".github/workflows/clean_validation_envs.yml" +BRANCH_WORKFLOW = ".github/workflows/deploy_tre_branch.yml" + + +def pending_work(ctx, members): + """Ignore only identified cleanup successors in the validated reference queue.""" + for member in members: + if member["status"] == "in_progress": + continue # verify_concurrency already proved this is the current cleanup owner. + run = github(ctx, f"/actions/runs/{member['run_id']}") + require( + isinstance(run, dict) + and run.get("id") == member["run_id"] + and isinstance(run.get("repository"), dict) + and run["repository"].get("full_name") == ctx["repository"], + "Cannot identify queued cleanup work.", + ) + if run.get("path") != CLEANUP_WORKFLOW or run.get("event") not in ("schedule", "workflow_dispatch"): + return True + if "job_id" not in member: + return True + job = github(ctx, f"/actions/jobs/{member['job_id']}") + if not ( + isinstance(job, dict) + and job.get("id") == member["job_id"] + and job.get("run_id") == member["run_id"] + and type(run.get("run_attempt")) is int + and run["run_attempt"] > 0 + and job.get("run_attempt") == run["run_attempt"] + and job.get("name") == "Clean " + ctx["ref"] + and job.get("status") in ACTIVE_STATUSES + ): + return True + return False + + +def timestamp(value): + require(isinstance(value, str), "Missing branch validation timestamp.") + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise RecoveryError("Invalid branch validation timestamp.") from error + require(result.tzinfo is not None, "Branch validation timestamp has no time zone.") + return result.timestamp() + + +def branch_activity(ctx, commit_time, now=None): + """Read all matching branch runs, including old runs with recent rerun activity.""" + require( + ctx["ref"].startswith("refs/heads/") and type(commit_time) is int and commit_time > 0, + "Invalid branch cleanup activity request.", + ) + now = datetime.now(timezone.utc).timestamp() if now is None else now + branch = ctx["ref"].removeprefix("refs/heads/") + latest = commit_time + runs, total = {}, None + for page in range(1, 12): + query = urlencode({"branch": branch, "event": "workflow_dispatch", "per_page": 100, "page": page}) + data = github(ctx, "/actions/workflows/deploy_tre_branch.yml/runs?" + query) + require( + isinstance(data, dict) + and type(data.get("total_count")) is int + and 0 <= data["total_count"] <= 1000 + and isinstance(data.get("workflow_runs"), list), + "Incomplete branch validation history.", + ) + total = data["total_count"] if total is None else total + require(total == data["total_count"], "Branch validation history changed during cleanup.") + for run in data["workflow_runs"]: + require( + isinstance(run, dict) + and type(run.get("id")) is int + and run["id"] > 0 + and run["id"] not in runs + and run.get("path") == BRANCH_WORKFLOW + and run.get("event") == "workflow_dispatch" + and run.get("head_branch") == branch + and all( + isinstance(run.get(key), dict) and run[key].get("full_name") == ctx["repository"] + for key in ("repository", "head_repository") + ), + "Invalid branch validation identity.", + ) + runs[run["id"]] = run + if run.get("status") in ACTIVE_STATUSES: + # Preparation can be active before the run joins the reference queue. + return now + require(run.get("status") == "completed", "Unknown branch validation status.") + created = timestamp(run.get("created_at")) + updated = timestamp(run.get("updated_at")) + require(created <= updated <= now, "Inconsistent branch validation activity time.") + # updated_at conservatively retains recent failed, cancelled and rerun attempts. + # It is an activity time, not a claim about exact job completion. + latest = max(latest, updated) + require(len(runs) <= total, "Inconsistent branch validation count.") + if len(runs) == total: + return latest + require(len(data["workflow_runs"]) == 100, "GitHub omitted branch validation runs.") + raise RecoveryError("Could not read all branch validation history.") + + +def mutation_allowed(ctx, members, commit_time=None, inactive_hours=None, now=None): + if pending_work(ctx, members): + return False + require((commit_time is None) == (inactive_hours is None), "Incomplete cleanup inactivity threshold.") + if commit_time is None: + # Closed PRs and deleted branches retain their orphan policy. The queue still applies. + return True + require( + ctx["ref"].startswith("refs/heads/") + and type(commit_time) is int + and commit_time > 0 + and type(inactive_hours) is int + and inactive_hours >= 0, + "Invalid branch cleanup inactivity threshold.", + ) + now = datetime.now(timezone.utc).timestamp() if now is None else now + latest = branch_activity(ctx, commit_time, now) + return (now - latest) // 3600 > inactive_hours diff --git a/devops/scripts/ci_cleanup_scope.py b/devops/scripts/ci_cleanup_scope.py index efec4160ab..031a2a64ad 100644 --- a/devops/scripts/ci_cleanup_scope.py +++ b/devops/scripts/ci_cleanup_scope.py @@ -8,6 +8,7 @@ import sys from ci_environment_id import validate_ref +from ci_cleanup_activity import branch_activity, mutation_allowed from recover_bootstrap_lease import RecoveryError, github, require, verify_concurrency @@ -124,7 +125,7 @@ def verify_lock(ctx): raise RecoveryError("Could not verify all cleanup jobs.") owners = [job for job in jobs.values() if job["name"] == expected_name and job.get("status") == "in_progress"] require(len(owners) == 1, "The current cleanup job is missing or ambiguous.") - verify_concurrency(ctx, job_id=owners[0]["id"]) + return verify_concurrency(ctx, job_id=owners[0]["id"]) def verify_target(groups, ref, group): @@ -142,19 +143,29 @@ def verify_target(groups, ref, group): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("operation", choices=("plan", "verify-lock", "verify-target")) + parser.add_argument("operation", choices=("plan", "verify-lock", "verify-target", "may-clean", "activity-time")) parser.add_argument("--group") + parser.add_argument("--commit-time", type=int) + parser.add_argument("--inactive-hours", type=int) args = parser.parse_args() try: if args.operation == "plan": print(json.dumps(references(inventory()), separators=(",", ":"))) else: ctx = context() - verify_lock(ctx) - if args.operation == "verify-target": + members = verify_lock(ctx) + if args.operation in ("verify-target", "may-clean"): verify_target(inventory(), ctx["ref"], args.group or "") + if args.operation == "may-clean": + allowed = mutation_allowed(ctx, members, args.commit_time, args.inactive_hours) + # Recheck ownership and queued work after reading history. + if allowed: + allowed = mutation_allowed(ctx, verify_lock(ctx)) + print("proceed" if allowed else "defer") else: verify_account() + if args.operation == "activity-time": + print(int(branch_activity(ctx, args.commit_time))) except (RecoveryError, ValueError, OSError, subprocess.SubprocessError) as error: print(f"ERROR: CI cleanup refused: {error}", file=sys.stderr) return 1 diff --git a/devops/scripts/clean_ci_validation_envs.sh b/devops/scripts/clean_ci_validation_envs.sh index 48ebe19932..2e6287d0a6 100755 --- a/devops/scripts/clean_ci_validation_envs.sh +++ b/devops/scripts/clean_ci_validation_envs.sh @@ -14,7 +14,13 @@ set -o nounset function stopEnv () { - python3 devops/scripts/ci_cleanup_scope.py verify-target --group "$1" + local decision + decision=$(python3 devops/scripts/ci_cleanup_scope.py may-clean --group "$1" "${@:2}") + if [[ "$decision" == defer ]]; then + echo "Validation activity defers stopping $1." + return 0 + fi + [[ "$decision" == proceed ]] || { echo "Invalid cleanup activity decision" >&2; exit 1; } local tre_rg="$1" if [[ "$tre_rg" == *-mgmt ]]; then echo "Management-only environment ${tre_rg} has no core services to stop. Keeping it until the destroy threshold." @@ -26,7 +32,13 @@ function stopEnv () function destroyEnv () { - python3 devops/scripts/ci_cleanup_scope.py verify-target --group "$1" + local decision + decision=$(python3 devops/scripts/ci_cleanup_scope.py may-clean --group "$1" "${@:2}") + if [[ "$decision" == defer ]]; then + echo "Validation activity defers destroying $1." + return 0 + fi + [[ "$decision" == proceed ]] || { echo "Invalid cleanup activity decision" >&2; exit 1; } # The destroy helper accepts the core name even when only management exists. devops/scripts/destroy_env_no_terraform.sh --core-tre-rg "${1%-mgmt}" } @@ -146,14 +158,16 @@ while read -r rg_name rg_ref_name; do # checking when was the last commit on the branch. last_commit_date_string=$(git for-each-ref --sort='-committerdate:iso8601' --format=' %(committerdate:iso8601)%09%(refname)' "${ref_in_remote}" | cut -f1) echo "Native ref is ${rg_ref_name}, last commit was on: ${last_commit_date_string}" - diff_in_hours=$(( ($(date +%s) - $(date -d "${last_commit_date_string}" +%s) )/(60*60) )) + last_commit_time=$(date -d "${last_commit_date_string}" +%s) + last_activity_time=$(python3 devops/scripts/ci_cleanup_scope.py activity-time --commit-time "$last_commit_time") + diff_in_hours=$(( ($(date +%s) - last_activity_time )/(60*60) )) if (( diff_in_hours > BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_DESTROY )); then echo "No recent activity on ${rg_ref_name}. Environment in ${rg_name} will be destroyed." - destroyEnv "${rg_name}" + destroyEnv "${rg_name}" --commit-time "$last_commit_time" --inactive-hours "$BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_DESTROY" elif (( diff_in_hours > BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_STOP )); then echo "No recent activity on ${rg_ref_name}. Environment in ${rg_name} will be stopped." - stopEnv "${rg_name}" + stopEnv "${rg_name}" --commit-time "$last_commit_time" --inactive-hours "$BRANCH_LAST_ACTIVITY_IN_HOURS_FOR_STOP" fi fi fi diff --git a/devops/scripts/recover_bootstrap_lease.py b/devops/scripts/recover_bootstrap_lease.py index 3300a13218..ca362123de 100644 --- a/devops/scripts/recover_bootstrap_lease.py +++ b/devops/scripts/recover_bootstrap_lease.py @@ -226,6 +226,7 @@ def verify_concurrency(ctx, job_id=None): and job.get("name") == "Deploy PR / Deploy Management", f"The concurrency job is not this recovery attempt's management deployment; target={target}.", ) + return members def verify_workflows(ctx, cache=None): diff --git a/devops/scripts/terraform_init.sh b/devops/scripts/terraform_init.sh index 096b1d65d0..642980b8d8 100755 --- a/devops/scripts/terraform_init.sh +++ b/devops/scripts/terraform_init.sh @@ -1,7 +1,7 @@ #!/bin/bash # Source this helper before initialising a management Terraform backend. -# Retry recognised probe permissions and workspace-listing failures before state locking. +# Retry recognised probe, workspace-listing and state-read permission failures. retry_with_backoff() { local func="$1" @@ -136,7 +136,14 @@ init_terraform() { echo "ERROR: Check the reported lock owner before attempting state recovery. No lock has been released by this helper." >&2 return 2 fi - # Retry only workspace-listing permission failures, not unknown state errors. + # Terraform's Azure backend returns state-read errors before locking or after + # releasing its lease. Lock and unlock failures must pass the guard above. + # Match the known storage code, not an unrelated 403 or unknown state error. + if grep -Fq 'Error loading state:' <<< "$terraform_output" \ + && grep -Fq 'AuthorizationPermissionMismatch' <<< "$terraform_output"; then + return 1 + fi + # Workspace listing also occurs before state locking. if grep -Fq 'Failed to get existing workspaces' <<< "$terraform_output" \ && is_storage_permission_error "$terraform_output"; then return 1 diff --git a/devops/scripts/verify_deployment_lock.py b/devops/scripts/verify_deployment_lock.py new file mode 100644 index 0000000000..1c7119dcec --- /dev/null +++ b/devops/scripts/verify_deployment_lock.py @@ -0,0 +1,81 @@ +"""Verify that a branch caller holds the deployment lock before Azure work starts.""" + +import os +import re +import sys + +from ci_environment_id import validate_ref +from recover_bootstrap_lease import RecoveryError, github, require, verify_concurrency + +ACTIVE_RUN_STATUSES = frozenset(("in_progress", "queued", "pending", "waiting", "requested")) + + +def context(): + require(os.environ.get("GITHUB_ACTIONS") == "true", "Delegated deployment requires GitHub Actions.") + ref = os.environ.get("CI_GIT_REF", "") + validate_ref(ref) + require( + ref.startswith("refs/heads/") and ref == os.environ.get("GITHUB_REF"), + "The delegated deployment reference differs from the branch caller.", + ) + repository = os.environ.get("GITHUB_REPOSITORY", "") + require( + re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+", repository) + and repository.split("/")[-1] not in (".", ".."), + "Invalid deployment repository.", + ) + values = [os.environ.get(key, "") for key in ("GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT")] + require( + all(value.isdecimal() and int(value) > 0 for value in values), "The deployment run identity is unavailable." + ) + sha = os.environ.get("GITHUB_SHA", "") + require(re.fullmatch(r"[0-9a-f]{40}", sha), "The deployment workflow commit is unavailable.") + return {"ref": ref, "repository": repository, "run_id": int(values[0]), "attempt": int(values[1]), "sha": sha} + + +def verify_delegated_lock(ctx): + run = github(ctx, f"/actions/runs/{ctx['run_id']}") + require( + isinstance(run, dict) + and type(run.get("id")) is int + and run["id"] == ctx["run_id"] + and type(run.get("run_attempt")) is int + and run["run_attempt"] == ctx["attempt"] + and run.get("status") in ACTIVE_RUN_STATUSES + and run.get("event") == "workflow_dispatch" + and run.get("path") == ".github/workflows/deploy_tre_branch.yml" + and run.get("head_branch") == ctx["ref"].removeprefix("refs/heads/") + and run.get("head_sha") == ctx["sha"] + and all( + isinstance(run.get(key), dict) and run[key].get("full_name") == ctx["repository"] + for key in ("repository", "head_repository") + ), + "The delegated deployment is not the current branch workflow run and attempt.", + ) + # The parent run can be queued while a job executes. The canonical group's + # in-progress owner, checked below, proves that this run holds the lock. + members = verify_concurrency(ctx) + # A reusable or job-level lease does not protect caller preparation. + require( + all("job_id" not in member for member in members if member["status"] == "in_progress"), + "The branch caller does not hold a workflow-level deployment lock.", + ) + + +def main(): + try: + delegated = os.environ.get("CI_DEPLOYMENT_LOCK_DELEGATED", "") + require(delegated in ("true", "false"), "The deployment lock delegation flag is invalid.") + if delegated == "false": + print("The reusable workflow holds the deployment reference lock.") + return 0 + verify_delegated_lock(context()) + print("The branch workflow owns the deployment reference lock for this run and attempt.") + except (RecoveryError, ValueError, OSError) as error: + print(f"ERROR: Deployment lock verification failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devops/tests/test_bootstrap.py b/devops/tests/test_bootstrap.py index 0182f7b547..98ce2639e3 100644 --- a/devops/tests/test_bootstrap.py +++ b/devops/tests/test_bootstrap.py @@ -31,6 +31,14 @@ If you want to change the default action to apply when no rule matches, please use 'az storage account update'. """ +# Diagnostic from fresh management deployment 38181301145, attempt 1. +STATE_READ_PERMISSION_ERROR = ( + "Error: Error loading state: executing request: unexpected status 403 " + "(403 This request is not authorized to perform this operation using this permission.) " + "with AuthorizationPermissionMismatch: This request is not authorized to perform " + "this operation using this permission." +) + MOCK_COMMAND = r""" import hashlib import json @@ -778,6 +786,44 @@ def test_unknown_init_permission_error_is_not_blindly_retried(self): self.assertIn("Error loading state: HTTP 403", self.output) self.assert_no_imports() + def test_state_read_permission_failure_after_probe_is_retried(self): + result = self.run_bootstrap( + init_responses=[ + {"code": 1, "output": STATE_READ_PERMISSION_ERROR}, + {"code": 0, "output": "Backend ready"}, + ] + ) + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(len(self.commands("terraform", "init")), 2) + self.assertEqual(self.sleeps(), [10]) + self.assertIn(STATE_READ_PERMISSION_ERROR, self.output) + + def test_state_read_permission_retries_are_bounded(self): + result = self.run_bootstrap(init_responses=[{"code": 1, "output": STATE_READ_PERMISSION_ERROR}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.commands("terraform", "init")), 6) + self.assertEqual(self.sleeps(), [10, 20, 40, 80, 160]) + self.assertEqual(self.output.count(STATE_READ_PERMISSION_ERROR), 6) + self.assert_no_imports() + + def test_state_read_permission_error_does_not_hide_lock_failure(self): + error = f"{STATE_READ_PERMISSION_ERROR}\nfailed to lock azure state: failed to write lock metadata" + result = self.run_bootstrap(init_responses=[{"code": 1, "output": error}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.commands("terraform", "init")), 1) + self.assertEqual(self.sleeps(), []) + self.assertIn(error, self.output) + self.assert_no_imports() + + def test_state_read_permission_error_does_not_hide_unlock_failure(self): + error = f"{STATE_READ_PERMISSION_ERROR}\nError unlocking Azure state. Lock ID: test-lock" + result = self.run_bootstrap(init_responses=[{"code": 1, "output": error}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.commands("terraform", "init")), 1) + self.assertEqual(self.sleeps(), []) + self.assertIn(error, self.output) + self.assert_no_imports() + def test_failed_unlock_with_403_is_visible_and_not_retried(self): error = ( "Error unlocking Azure state. Lock ID: test-lock\nError: failed to delete lock info from metadata: HTTP 403" @@ -928,6 +974,35 @@ def test_existing_lock_is_not_retried(self): self.assertEqual(self.sleeps(), []) self.assert_no_deployment() + def test_state_read_permission_failure_is_retried_before_plan(self): + result = self.run_deploy( + init_responses=[ + {"code": 1, "output": STATE_READ_PERMISSION_ERROR}, + {"code": 0, "output": "Backend ready"}, + ] + ) + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(len(self.commands("terraform", "init")), 2) + self.assertEqual(self.sleeps(), [10]) + self.assertIn(STATE_READ_PERMISSION_ERROR, self.output) + self.assert_deployed_once() + + def test_state_read_permission_exhaustion_stops_before_plan(self): + result = self.run_deploy(init_responses=[{"code": 1, "output": STATE_READ_PERMISSION_ERROR}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.commands("terraform", "init")), 6) + self.assertEqual(self.sleeps(), [10, 20, 40, 80, 160]) + self.assert_no_deployment() + + def test_state_read_permission_error_does_not_hide_unlock_failure(self): + error = f"{STATE_READ_PERMISSION_ERROR}\nError unlocking Azure state. Lock ID: test-lock" + result = self.run_deploy(init_responses=[{"code": 1, "output": error}]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.commands("terraform", "init")), 1) + self.assertEqual(self.sleeps(), []) + self.assertIn(error, self.output) + self.assert_no_deployment() + def test_state_permission_error_is_not_retried(self): error = "Error loading state: HTTP 403" result = self.run_deploy(init_responses=[{"code": 1, "output": error}]) diff --git a/devops/tests/test_ci_cleanup_activity.py b/devops/tests/test_ci_cleanup_activity.py new file mode 100644 index 0000000000..6464255cab --- /dev/null +++ b/devops/tests/test_ci_cleanup_activity.py @@ -0,0 +1,178 @@ +"""Protect recently used environments without retaining unrelated or orphaned groups.""" + +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +import sys +import unittest +from unittest.mock import patch +from urllib.parse import parse_qs, urlsplit + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import ci_cleanup_activity as activity # noqa: E402 + + +class ActivityTests(unittest.TestCase): + def setUp(self): + self.now = datetime(2026, 10, 11, 12, tzinfo=timezone.utc).timestamp() + self.ctx = {"repository": "microsoft/AzureTRE", "ref": "refs/heads/feature/test", "run_id": 123} + self.members = [{"run_id": 123, "job_id": 10, "status": "in_progress"}] + self.runs = [] + self.requests = [] + + def run_record(self, number=200, age_hours=1, **kwargs): + return { + "id": number, + "path": activity.BRANCH_WORKFLOW, + "event": "workflow_dispatch", + "head_branch": "feature/test", + "head_sha": "a" * 40, + "repository": {"full_name": self.ctx["repository"]}, + "head_repository": {"full_name": self.ctx["repository"]}, + "status": "completed", + "conclusion": "success", + "created_at": "2026-09-01T00:00:00Z", + "updated_at": datetime.fromtimestamp(self.now - age_hours * 3600, timezone.utc).isoformat(), + **kwargs, + } + + def github(self, ctx, suffix): + self.requests.append(suffix) + self.assertEqual(ctx, self.ctx) + self.assertTrue(suffix.startswith("/actions/workflows/deploy_tre_branch.yml/runs?")) + query = parse_qs(urlsplit(suffix).query) + self.assertEqual(query["branch"], ["feature/test"]) + self.assertEqual(query["event"], ["workflow_dispatch"]) + self.assertNotIn("created", query) # Older runs can have a recent rerun. + page = int(query["page"][0]) + return {"total_count": len(self.runs), "workflow_runs": deepcopy(self.runs[(page - 1) * 100 : page * 100])} # noqa: E203 + + def allowed(self, age=100, threshold=4): + with patch.object(activity, "github", side_effect=self.github): + return activity.mutation_allowed(self.ctx, self.members, int(self.now - age * 3600), threshold, self.now) + + def test_old_commit_and_recent_success_failure_or_cancellation_are_retained(self): + for conclusion in ("success", "failure", "cancelled", "timed_out"): + with self.subTest(conclusion=conclusion): + self.runs = [self.run_record(conclusion=conclusion, age_hours=1 / 60)] + self.assertFalse(self.allowed()) + + def test_recent_rerun_of_old_sha_extends_activity(self): + self.runs = [self.run_record(run_attempt=4, head_sha="b" * 40)] + self.assertFalse(self.allowed()) + + def test_preparation_and_pending_runs_are_active_without_queue_membership(self): + for status in activity.ACTIVE_STATUSES: + with self.subTest(status=status): + self.runs = [self.run_record(status=status, age_hours=100)] + self.assertFalse(self.allowed()) + + def test_old_activity_preserves_thresholds_and_new_commits_extend_activity(self): + self.runs = [self.run_record(age_hours=6)] + self.assertTrue(self.allowed(threshold=4)) + self.assertFalse(self.allowed(threshold=48)) + self.assertFalse(self.allowed(age=1)) + self.runs = [self.run_record(age_hours=60)] + self.assertTrue(self.allowed(threshold=48)) + self.runs = [] + self.assertTrue(self.allowed(threshold=48)) + + def test_second_page_recent_run_prevents_destruction(self): + self.runs = [self.run_record(number=n, age_hours=100) for n in range(200, 300)] + [self.run_record(number=300)] + self.assertFalse(self.allowed(threshold=48)) + self.assertTrue(self.requests[-1].endswith("page=2")) + + def test_unrelated_history_is_not_requested(self): + self.runs = [] + self.assertTrue(self.allowed()) + self.assertEqual(len(self.requests), 1) + self.assertIn("branch=feature%2Ftest", self.requests[0]) + + def test_unexpected_identity_status_or_time_refuses_cleanup(self): + for change in ( + {"id": True}, + {"path": activity.CLEANUP_WORKFLOW}, + {"event": "issue_comment"}, + {"head_branch": "other"}, + {"repository": {"full_name": "other/repo"}}, + {"head_repository": None}, + {"status": "unknown"}, + {"updated_at": None}, + {"updated_at": "not-a-date"}, + {"updated_at": "2026-10-11T11:00:00"}, + {"updated_at": "2026-10-12T11:00:00Z"}, + {"updated_at": "2026-08-01T11:00:00Z"}, + ): + with self.subTest(change=change), self.assertRaises(activity.RecoveryError): + self.runs = [self.run_record(age_hours=100, **change)] + self.allowed() + + def test_incomplete_duplicate_or_unbounded_history_refuses_cleanup(self): + for response in ( + {}, + {"total_count": 1001, "workflow_runs": []}, + {"total_count": True, "workflow_runs": []}, + {"total_count": 2, "workflow_runs": [self.run_record(age_hours=100)]}, + {"total_count": 2, "workflow_runs": [self.run_record(age_hours=100)] * 2}, + {"total_count": 0, "workflow_runs": [self.run_record(age_hours=100)]}, + ): + with ( + self.subTest(response=response), + patch.object(activity, "github", return_value=response), + self.assertRaises(activity.RecoveryError), + ): + activity.mutation_allowed(self.ctx, self.members, int(self.now - 100 * 3600), 4, self.now) + + def test_changed_page_count_and_api_error_refuse_cleanup(self): + first = { + "total_count": 101, + "workflow_runs": [self.run_record(number=n, age_hours=100) for n in range(200, 300)], + } + for responses in ([first, {"total_count": 102, "workflow_runs": []}], activity.RecoveryError("rate limit")): + with patch.object(activity, "github", side_effect=responses), self.assertRaises(activity.RecoveryError): + activity.mutation_allowed(self.ctx, self.members, int(self.now - 100 * 3600), 4, self.now) + + def test_queued_deployment_or_unknown_work_defers_even_orphan_deletion(self): + self.members.append({"run_id": 456, "status": "pending"}) + for path in (activity.BRANCH_WORKFLOW, ".github/workflows/pr_comment_bot.yml", None, activity.CLEANUP_WORKFLOW): + run = {"id": 456, "path": path, "event": "schedule", "repository": {"full_name": self.ctx["repository"]}} + with patch.object(activity, "github", return_value=run): + self.assertFalse(activity.mutation_allowed(self.ctx, self.members)) + + def test_verified_cleanup_successors_do_not_extend_activity(self): + self.members.append({"run_id": 456, "job_id": 50, "status": "pending"}) + run = { + "id": 456, + "path": activity.CLEANUP_WORKFLOW, + "event": "schedule", + "run_attempt": 2, + "repository": {"full_name": self.ctx["repository"]}, + } + job = {"id": 50, "run_id": 456, "run_attempt": 2, "name": "Clean " + self.ctx["ref"], "status": "pending"} + with patch.object(activity, "github", side_effect=[run, job]): + self.assertTrue(activity.mutation_allowed(self.ctx, self.members)) + for change in ({"run_attempt": 1}, {"name": "Unidentified"}, {"id": 60}, {"status": "completed"}): + with patch.object(activity, "github", side_effect=[run, {**job, **change}]): + self.assertFalse(activity.mutation_allowed(self.ctx, self.members)) + + def test_unverified_queue_identity_refuses_cleanup(self): + self.members.append({"run_id": 456, "job_id": 50, "status": "pending"}) + with patch.object(activity, "github", return_value={}), self.assertRaises(activity.RecoveryError): + activity.mutation_allowed(self.ctx, self.members) + + def test_orphan_decision_has_no_completed_run_grace_period(self): + with patch.object(activity, "github") as github: + self.assertTrue(activity.mutation_allowed(self.ctx, self.members)) + github.assert_not_called() + + def test_partial_or_invalid_age_requests_refuse_cleanup(self): + for commit, hours in ((None, 4), (1, None), (-1, 4), (1, -1), (True, 4)): + with self.subTest(commit=commit, hours=hours), self.assertRaises(activity.RecoveryError): + activity.mutation_allowed(self.ctx, self.members, commit, hours, self.now) + self.ctx["ref"] = "refs/pull/123/merge" + with self.assertRaises(activity.RecoveryError): + activity.mutation_allowed(self.ctx, self.members, int(self.now), 4, self.now) + + +if __name__ == "__main__": + unittest.main() diff --git a/devops/tests/test_ci_cleanup_scope.py b/devops/tests/test_ci_cleanup_scope.py index cad8f9c744..112d893b63 100644 --- a/devops/tests/test_ci_cleanup_scope.py +++ b/devops/tests/test_ci_cleanup_scope.py @@ -1,7 +1,9 @@ """Verify cleanup ownership, pagination and the shared workflow lock contract.""" from copy import deepcopy +from contextlib import redirect_stderr, redirect_stdout import importlib.util +import io import json import os from pathlib import Path @@ -15,6 +17,7 @@ SPEC = importlib.util.spec_from_file_location("ci_cleanup_scope", SCRIPTS / "ci_cleanup_scope.py") scope = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(scope) +import ci_cleanup_activity as activity # noqa: E402 class ScopeTests(unittest.TestCase): @@ -125,6 +128,37 @@ def test_rate_limit_or_lock_failure_never_allows_cleanup(self): with self.assertRaises(scope.RecoveryError): scope.verify_lock(self.ctx) + def test_new_queued_deployment_defers_command_after_history_check(self): + owner = [{"run_id": 123, "job_id": 10, "status": "in_progress"}] + # The updated caller queues the whole run before Preparation starts. + successor = {"run_id": 456, "status": "pending"} + run = {"id": 456, "path": activity.BRANCH_WORKFLOW, "repository": {"full_name": self.ctx["repository"]}} + output = io.StringIO() + with ( + patch.object(sys, "argv", ["ci_cleanup_scope.py", "may-clean", "--group", "rg-tretest"]), + patch.object(scope, "verify_lock", side_effect=[owner, owner + [successor]]) as lock, + patch.object(scope, "inventory", return_value={"rg-tretest": self.ctx["ref"]}), + patch.object(activity, "github", return_value=run), + redirect_stdout(output), + ): + self.assertEqual(scope.main(), 0) + self.assertEqual(lock.call_count, 2) + self.assertEqual(output.getvalue(), "defer\n") + + def test_lost_second_lock_fails_command_without_permission_to_mutate(self): + owner = [{"run_id": 123, "job_id": 10, "status": "in_progress"}] + output, error = io.StringIO(), io.StringIO() + with ( + patch.object(sys, "argv", ["ci_cleanup_scope.py", "may-clean", "--group", "rg-tretest"]), + patch.object(scope, "verify_lock", side_effect=[owner, scope.RecoveryError("owner changed")]), + patch.object(scope, "inventory", return_value={"rg-tretest": self.ctx["ref"]}), + redirect_stdout(output), + redirect_stderr(error), + ): + self.assertEqual(scope.main(), 1) + self.assertEqual(output.getvalue(), "") + self.assertIn("owner changed", error.getvalue()) + def test_unscoped_non_ci_and_invalid_references_are_refused(self): for key, value in ( ("CI_CLEANUP_REF", ""), @@ -203,7 +237,7 @@ class WorkflowContractTests(unittest.TestCase): def test_all_supported_writers_share_the_reference_group(self): workflows = SCRIPTS.parents[1] / ".github/workflows" deploy = (workflows / "deploy_tre_reusable.yml").read_text() - self.assertIn("group: deploy-${{ inputs.ciGitRef }}", deploy) + self.assertIn("|| format('deploy-{0}', inputs.ciGitRef)", deploy) self.assertIn("CI_GIT_REF: ${{ inputs.ciGitRef }}", deploy) explicit = (workflows / "pr_comment_bot.yml").read_text() for name in ("ciGitRef", "branchCiGitRef"): @@ -220,7 +254,7 @@ def test_cleanup_waits_until_all_resource_group_deletions_complete(self): self.assertNotIn("--no-wait", script) self.assertNotIn("actions/runs?", script) self.assertLess(script.index("ci_cleanup_scope.py verify-lock"), script.index("az group list")) - self.assertRegex(script, re.compile(r"function destroyEnv.*?verify-target.*?destroy_env_no_terraform", re.S)) + self.assertRegex(script, re.compile(r"function destroyEnv.*?may-clean.*?destroy_env_no_terraform", re.S)) def test_cleanup_job_keeps_the_lock_for_long_deletions(self): scheduled = (SCRIPTS.parents[1] / ".github/workflows/clean_validation_envs.yml").read_text() diff --git a/devops/tests/test_clean_ci_validation_envs.py b/devops/tests/test_clean_ci_validation_envs.py index b011e29af0..52bdd29712 100644 --- a/devops/tests/test_clean_ci_validation_envs.py +++ b/devops/tests/test_clean_ci_validation_envs.py @@ -29,6 +29,12 @@ if command == "python3": if config.get("guard_error") or not os.environ.get("CI_CLEANUP_REF"): sys.exit("CI cleanup refused: exclusive ownership unavailable") + if "may-clean" in args: + if config.get("activity_error"): + sys.exit("CI cleanup refused: history incomplete") + print(config.get("activity_decision", "proceed")) + elif "activity-time" in args: + print(2000000000 - config.get("activity_age_hours", config["age_hours"]) * 3600) sys.exit(0) elif command == "gh": if args[0] == "api": @@ -257,6 +263,40 @@ def test_scoped_cleanup_ignores_unrelated_workflows_and_other_targets(self): self.assertEqual(self.environment_actions(), [["destroy_env_no_terraform.sh", "--core-tre-rg", "rg-tretest"]]) self.assertFalse(any(call[0] == "gh" and any("actions/runs?" in item for item in call) for call in self.calls)) + def test_native_branch_uses_latest_activity_for_stop_and_destroy(self): + self.config.update(groups=["rg-tretest\trefs/heads/feature/test"], age_hours=100) + for hours, action in ((1, None), (6, "control_tre.sh"), (60, "destroy_env_no_terraform.sh")): + with self.subTest(hours=hours): + self.config["activity_age_hours"] = hours + result = self.run_cleanup() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([call[0] for call in self.environment_actions()], [] if action is None else [action]) + if action: + guards = [call for call in self.calls if "may-clean" in call] + self.assertEqual(len(guards), 1) + self.assertIn("--commit-time", guards[0]) + self.assertIn("--inactive-hours", guards[0]) + + def test_activity_recheck_defers_old_and_orphaned_environments(self): + for ref, missing, prs in ( + ("refs/pull/5085/merge", False, self.config["open_prs"]), + ("refs/pull/5085/merge", False, []), + ("refs/heads/feature/deleted", True, []), + ): + with self.subTest(ref=ref, missing=missing, prs=prs): + self.config.update( + groups=["rg-tretest\t" + ref], missing_branch=missing, open_prs=prs, activity_decision="defer" + ) + result = self.run_cleanup() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.environment_actions(), []) + + def test_activity_api_failure_prevents_mutation(self): + self.config["activity_error"] = True + result = self.run_cleanup() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.environment_actions(), []) + def test_main_workspace_cleanup_is_separate_and_synchronous(self): self.config["selected_ref"] = True self.env["CI_CLEANUP_REF"] = "refs/heads/main" diff --git a/devops/tests/test_core_outputs.py b/devops/tests/test_core_outputs.py new file mode 100644 index 0000000000..c0c9bdbd5a --- /dev/null +++ b/devops/tests/test_core_outputs.py @@ -0,0 +1,367 @@ +"""Exercise core output failures through the real UI Make target, without Azure.""" + +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +BASH = shutil.which("bash") +OUTPUT_NAMES = re.findall(r'^output "([^"]+)"', (ROOT / "core/terraform/outputs.tf").read_text(), re.MULTILINE) +VALID_OUTPUT = json.dumps( + { + name: {"value": "mockstorage" if name == "static_web_storage" else f"mock-{name}", "type": "string"} + for name in OUTPUT_NAMES + } +) +AUXILIARY_OUTPUTS = { + "WORKSPACE_API_CLIENT_ID": ("TEST_WORKSPACE_APP_ID",), + "WORKSPACE_API_CLIENT_SECRET": ("TEST_WORKSPACE_APP_SECRET",), + "SUB_ID": ("SUBSCRIPTION_ID", "AZURE_SUBSCRIPTION_ID"), + "TENANT_ID": ("AZURE_TENANT_ID",), +} + + +class CoreOutputTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + for relative in ( + "Makefile", + "config_schema.json", + "core/terraform/outputs.sh", + "core/terraform/json-to-env.sh", + "devops/scripts/load_env.sh", + "devops/scripts/load_and_validate_env.sh", + "devops/scripts/construct_tre_url.sh", + "devops/scripts/convert_azure_env_to_arm_env.sh", + ): + target = self.root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / relative, target) + self.core = self.root / "core" + self.log = self.root / "calls.log" + scripts = self.root / "devops/scripts" + scripts.mkdir(parents=True, exist_ok=True) + (scripts / "bootstrap_azure_env.sh").write_text("true\n") + (scripts / "storage_enable_public_access.sh").write_text( + 'echo storage-enable >> "$CALL_LOG"\ntrap \'echo storage-disable >> "$CALL_LOG"\' EXIT\n' + ) + deploy = scripts / "build_deploy_ui.sh" + deploy.write_text('#!/bin/bash\necho "ui:$STORAGE_ACCOUNT" >> "$CALL_LOG"\n') + deploy.chmod(0o755) + mock = self.root / "bin" + mock.mkdir() + terraform = mock / "terraform" + terraform.write_text( + '#!/bin/bash\necho "terraform:$1" >> "$CALL_LOG"\n' + 'if [ "$1" = init ]; then exit "${INIT_EXIT:-0}"; fi\n' + 'printf "%s" "$OUTPUT_JSON"\n' + 'if [ "${OUTPUT_EXIT:-0}" != 0 ]; then echo "Failed to load state" >&2; fi\n' + 'exit "${OUTPUT_EXIT:-0}"\n' + ) + terraform.chmod(0o755) + self.env = { + **os.environ, + "PATH": str(mock) + os.pathsep + os.environ["PATH"], + "CALL_LOG": str(self.log), + "OUTPUT_JSON": VALID_OUTPUT, + "INIT_EXIT": "0", + "OUTPUT_EXIT": "0", + "DEPLOY_UI": "true", + **dict.fromkeys(AUXILIARY_OUTPUTS, ""), + } + self.private = self.core / "private.env" + self.private.write_text("EXISTING='keep me'\n") + self.cache = self.core / "tre_output.json" + + def run_make(self): + result = subprocess.run( + [shutil.which("make"), "--no-print-directory", f"SHELL={BASH}", "build-and-deploy-ui"], + cwd=self.root, + env=self.env, + capture_output=True, + text=True, + timeout=10, + ) + self.calls = self.log.read_text().splitlines() + self.assertEqual(self.calls[-1], "storage-disable") + self.assertEqual(list(self.core.glob("*.tmp.*")), []) + return result + + def assert_stopped(self, result): + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse(any(call.startswith("ui:") for call in self.calls)) + self.assertEqual(self.private.read_text(), "EXISTING='keep me'\n") + + def test_failed_init_stops_before_output_and_ui(self): + self.env["INIT_EXIT"] = "17" + result = self.run_make() + self.assert_stopped(result) + self.assertNotIn("terraform:output", self.calls) + self.assertFalse(self.cache.exists()) + self.assertIn("Error 17", result.stderr) + + def test_failed_output_stops_ui_and_does_not_cache_partial_output(self): + for output in ("", '{"partial":', VALID_OUTPUT): + with self.subTest(output=output): + self.env.update(OUTPUT_EXIT="23", OUTPUT_JSON=output) + result = self.run_make() + self.assert_stopped(result) + self.assertFalse(self.cache.exists()) + self.assertIn("Failed to load state", result.stderr) + self.assertIn("Error 23", result.stderr) + + def test_invalid_successful_output_is_not_cached(self): + for output in ("", "{}", "[]", "null", '{"partial":'): + with self.subTest(output=output): + self.env["OUTPUT_JSON"] = output + result = self.run_make() + self.assert_stopped(result) + self.assertFalse(self.cache.exists()) + + def test_invalid_nonempty_cache_stops_ui(self): + self.cache.write_text('{"partial":') + result = self.run_make() + self.assert_stopped(result) + self.assertNotIn("terraform:output", self.calls) + + def test_incomplete_outputs_are_not_published(self): + incomplete_outputs = [{"unexpected": {"value": "x"}}, {"static_web_storage": {"value": "mockstorage"}}] + for missing in ("static_web_storage", "core_resource_group_name", "keyvault_name"): + output = json.loads(VALID_OUTPUT) + del output[missing] + incomplete_outputs.append(output) + for output in incomplete_outputs: + with self.subTest(output=output): + self.env["OUTPUT_JSON"] = json.dumps(output) + self.assert_stopped(self.run_make()) + self.assertFalse(self.cache.exists()) + + def test_invalid_mapped_values_are_not_published(self): + for value in ( + None, + "", + " \t\n", + "mock\nstorage", + "mock\rstorage", + "\x00mockstorage", + "mock\x00storage", + "mockstorage\x00", + "mock'storage", + 1, + False, + [], + {}, + ): + with self.subTest(value=value): + output = json.loads(VALID_OUTPUT) + output["static_web_storage"]["value"] = value + self.env["OUTPUT_JSON"] = json.dumps(output) + self.assert_stopped(self.run_make()) + self.assertFalse(self.cache.exists()) + + def test_missing_output_value_is_not_published(self): + output = json.loads(VALID_OUTPUT) + output["static_web_storage"] = {"type": "string"} + self.env["OUTPUT_JSON"] = json.dumps(output) + self.assert_stopped(self.run_make()) + self.assertFalse(self.cache.exists()) + + def test_invalid_auxiliary_values_preserve_existing_files(self): + marker = self.root / "injected" + invalid_values = ("mock'secret", "mock\nsecret", "mock\rsecret", f"secret'; touch {marker}; #") + for source_name in AUXILIARY_OUTPUTS: + for value in invalid_values: + for cached in (False, True): + with self.subTest(source_name=source_name, value=value, cached=cached): + self.env[source_name] = value + if cached: + self.cache.write_text(VALID_OUTPUT) + else: + self.cache.unlink(missing_ok=True) + result = self.run_make() + self.assert_stopped(result) + self.assertIn(source_name, result.stderr) + self.assertNotIn(value, result.stdout + result.stderr) + self.assertFalse(marker.exists()) + if cached: + self.assertEqual(self.cache.read_text(), VALID_OUTPUT) + else: + self.assertFalse(self.cache.exists()) + self.env[source_name] = "" + + def test_auxiliary_values_from_core_env_are_validated(self): + value = "mock'secret" + (self.core / ".env").write_text(f"WORKSPACE_API_CLIENT_SECRET={shlex.quote(value)}\n") + self.assert_stopped(self.run_make()) + self.assertFalse(self.cache.exists()) + + @unittest.skipUnless(shutil.which("pajv"), "The development container provides pajv for config validation") + def test_config_rejects_unsupported_secrets_before_shell_export(self): + config = { + "location": "westeurope", + "tre_id": "mocktre", + "authentication": { + "aad_tenant_id": "00000000-0000-0000-0000-000000000000", + "workspace_api_client_secret": "mock-safe-secret", + }, + } + config_file = self.root / "config.yaml" + config_file.write_text(json.dumps(config)) + valid = subprocess.run( + [shutil.which("pajv"), "validate", "-s", "config_schema.json", "-d", "config.yaml"], + cwd=self.root, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(valid.returncode, 0, valid.stdout + valid.stderr) + yq = self.root / "bin/yq" + yq.write_text('#!/bin/bash\necho yq >> "$CALL_LOG"\nexit 97\n') + yq.chmod(0o755) + for character in ("'", "\r", "\n", "\x00"): + for value in (f"mock{character}secret-value", f"mock-secret-value{character}"): + with self.subTest(value=value): + config["authentication"]["workspace_api_client_secret"] = value + config_file.write_text(json.dumps(config)) + result = subprocess.run( + [BASH, "-c", '. "$DIR/load_and_validate_env.sh"'], + cwd=self.root, + env={**self.env, "DIR": str(self.root / "devops/scripts")}, + capture_output=True, + text=True, + timeout=10, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("config.yaml is invalid", result.stdout) + self.assertNotIn(value, result.stdout + result.stderr) + self.assertFalse(self.log.exists(), "Config validation must fail before yq exports values") + self.assertEqual(self.private.read_text(), "EXISTING='keep me'\n") + + def test_auxiliary_values_match_both_environment_loaders(self): + output_names = [name for names in AUXILIARY_OUTPUTS.values() for name in names] + print_values = 'printf "%s\\0" ' + " ".join(f'"${{{name}}}"' for name in output_names) + for value in ("", ' \tmock"\\$secret; `printf injected`; $(printf injected) == \t '): + with self.subTest(value=value): + self.env.update(dict.fromkeys(AUXILIARY_OUTPUTS, value)) + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + for loader in ("load_env.sh", "load_and_validate_env.sh"): + with self.subTest(loader=loader): + result = subprocess.run( + [ + BASH, + "-c", + '. "$1" "$2"; ' + print_values, + "bash", + str(self.root / "devops/scripts" / loader), + str(self.private), + ], + cwd=self.root, + env={ + **self.env, + "DIR": str(self.root / "devops/scripts"), + "USE_ENV_VARS_NOT_FILES": "true", + }, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.split("\x00"), [value] * len(output_names) + [""]) + + def test_incomplete_cache_preserves_existing_files(self): + invalid_cache = '{"unexpected":{"value":"x"}}' + self.cache.write_text(invalid_cache) + self.assert_stopped(self.run_make()) + self.assertEqual(self.cache.read_text(), invalid_cache) + self.assertNotIn("terraform:output", self.calls) + + def test_multiple_output_documents_are_not_published(self): + self.env["OUTPUT_JSON"] = VALID_OUTPUT + "\n" + VALID_OUTPUT + self.assert_stopped(self.run_make()) + self.assertFalse(self.cache.exists()) + + def test_successful_output_reaches_ui_with_storage_account(self): + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("ui:mockstorage", self.calls) + self.assertEqual(json.loads(self.cache.read_text()), json.loads(VALID_OUTPUT)) + self.assertIn("STORAGE_ACCOUNT='mockstorage'", self.private.read_text()) + + def test_valid_cache_avoids_terraform(self): + self.cache.write_text(VALID_OUTPUT) + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call.startswith("terraform:") for call in self.calls)) + self.assertIn("ui:mockstorage", self.calls) + + def test_legacy_optional_output_is_preserved_when_present(self): + output = json.loads(VALID_OUTPUT) + output["airlock_malware_scan_result_topic_name"] = {"value": "legacy-topic"} + self.cache.write_text(json.dumps(output)) + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("AIRLOCK_MALWARE_SCAN_RESULT_TOPIC_NAME='legacy-topic'", self.private.read_text()) + + def test_output_values_match_environment_loaders(self): + output = json.loads(VALID_OUTPUT) + value = 'mock"\\$storage; echo injected' + output["static_web_storage"]["value"] = value + self.env["OUTPUT_JSON"] = json.dumps(output) + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"ui:{value}", self.calls) + self.assertNotIn("injected", result.stdout) + # load_and_validate_env.sh sources private.env directly. + sourced = subprocess.run( + [BASH, "-c", '. "$1"; printf "%s" "$STORAGE_ACCOUNT"', "bash", str(self.private)], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(sourced.returncode, 0, sourced.stderr) + self.assertEqual(sourced.stdout, value) + + def test_empty_cache_is_regenerated(self): + self.cache.write_text("") + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("terraform:output", self.calls) + self.assertIn("ui:mockstorage", self.calls) + + def test_conversion_failure_preserves_environment_and_retries(self): + output = json.loads(VALID_OUTPUT) + output["static_web_storage"] = 1 + self.env["OUTPUT_JSON"] = json.dumps(output) + result = self.run_make() + self.assert_stopped(result) + self.assertFalse(self.cache.exists()) + terraform_outputs_before_retry = self.calls.count("terraform:output") + + self.env["OUTPUT_JSON"] = VALID_OUTPUT + retry = self.run_make() + self.assertEqual(retry.returncode, 0, retry.stderr) + self.assertEqual(self.calls.count("terraform:output"), terraform_outputs_before_retry + 1) + self.assertIn("ui:mockstorage", self.calls) + + def test_converter_reports_invalid_json(self): + for output in ("", "{}", "[]", "null", '{"partial":', VALID_OUTPUT + "\n" + VALID_OUTPUT): + with self.subTest(output=output): + result = subprocess.run( + [BASH, "core/terraform/json-to-env.sh"], + cwd=self.root, + input=output, + capture_output=True, + text=True, + timeout=10, + ) + self.assertNotEqual(result.returncode, 0) diff --git a/devops/tests/test_mgmt_storage_private_endpoint.py b/devops/tests/test_mgmt_storage_private_endpoint.py new file mode 100644 index 0000000000..5b5d70c154 --- /dev/null +++ b/devops/tests/test_mgmt_storage_private_endpoint.py @@ -0,0 +1,262 @@ +"""Exercise the core management storage private endpoint step with mocked tools.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[2] / "core/terraform/mgmt_storage_private_endpoint.sh" +PE_ADDRESS = "module.resource_processor_vmss_porter[0].azurerm_private_endpoint.mgmtblobpe" +HOST = "mockstorage.blob.core.windows.net" + +MOCK_COMMAND = r""" +import json +import os +from pathlib import Path +import sys + +root = Path(os.environ["MOCK_ROOT"]) +config = json.loads((root / "config.json").read_text()) +state_file = root / "state.json" +state = json.loads(state_file.read_text()) if state_file.exists() else {"lookups": 0} +args = sys.argv[1:] +command = Path(sys.argv[0]).name +with (root / "calls.jsonl").open("a") as log: + log.write(json.dumps([command, *args]) + "\n") + + +def finish(code=0, output=""): + state_file.write_text(json.dumps(state)) + if output: + print(output, file=sys.stderr if code else sys.stdout) + sys.exit(code) + + +if command == "sleep": + finish() +if command == "timeout": + os.execvp(args[1], args[1:]) +if command == "az" and args[:3] == ["storage", "account", "show"]: + finish(output="https://" + config["host"] + "/") +if command == "az" and args[:3] == ["storage", "blob", "exists"]: + ready = config["backend"][min(state["lookups"], len(config["backend"])) - 1] + finish(0 if ready else 1, config["exists"] if ready else config["backend_error"]) +if command == "terraform" and args[0] == "plan": + finish(config["plan_exit"]) +if command == "terraform" and args[0] == "apply": + finish(config["apply_exit"], "apply failed" if config["apply_exit"] else "") +if command == "dig": + if "CNAME" not in args: + finish(output="DNS diagnostics") + state["lookups"] += 1 + alias = config["cnames"][min(state["lookups"], len(config["cnames"])) - 1] + finish(output=alias) +if command == "getent": + if state["lookups"] == 0: + finish(output="20.0.0.1 STREAM " + args[1]) + answer = config["dns"][min(state["lookups"], len(config["dns"])) - 1] + if answer: + finish(output="10.1.2.3 STREAM " + args[1]) + finish(2) +finish(99, "Unexpected mocked command: " + repr([command, *args])) +""" + + +class MgmtStoragePrivateEndpointTests(unittest.TestCase): + def run_script( + self, + plan_exit=2, + dns=(True,), + apply_exit=0, + rp_type="vmss_porter", + cnames=None, + backend=(True,), + host=HOST, + exists="true", + backend_error="backend unavailable", + ): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "config.json").write_text( + json.dumps( + { + "plan_exit": plan_exit, + "dns": list(dns), + "apply_exit": apply_exit, + "cnames": cnames or [host.replace(".", ".privatelink.", 1) + "."], + "backend": backend, + "host": host, + "exists": exists, + "backend_error": backend_error, + } + ) + ) + for name in ("az", "terraform", "getent", "sleep", "dig", "timeout"): + executable = root / name + executable.write_text(f"#!{sys.executable}\n" + MOCK_COMMAND) + executable.chmod(0o755) + env = { + **os.environ, + "MOCK_ROOT": str(root), + "PATH": str(root) + os.pathsep + os.environ["PATH"], + "TF_VAR_mgmt_storage_account_name": "mockstorage", + "TF_VAR_mgmt_resource_group_name": "mockgroup", + "TF_VAR_terraform_state_container_name": "tfstate", + "TRE_ID": "mock-tre", + "TF_VAR_resource_processor_type": rp_type, + } + result = subprocess.run( + ["/bin/bash", str(SCRIPT)], + cwd=root, + env=env, + text=True, + capture_output=True, + timeout=60, + ) + calls_file = root / "calls.jsonl" + self.calls = ( + [json.loads(line) for line in calls_file.read_text().splitlines()] if calls_file.exists() else [] + ) + self.output = result.stdout + result.stderr + return result + + def terraform_calls(self, subcommand): + return [c for c in self.calls if c[0] == "terraform" and c[1] == subcommand] + + def dns_attempts(self): + return [c for c in self.calls if c[0] == "dig" and "CNAME" in c] + + def blob_read_calls(self): + return [c for c in self.calls if c[:4] == ["az", "storage", "blob", "exists"]] + + def test_unchanged_private_endpoint_skips_apply_but_checks_readiness(self): + result = self.run_script(plan_exit=0) + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(len(self.terraform_calls("plan")), 1) + self.assertEqual(self.terraform_calls("apply"), []) + self.assertEqual(len(self.dns_attempts()), 3) + self.assertEqual(len(self.blob_read_calls()), 3) + self.assertEqual(self.terraform_calls("state"), []) + for call in self.blob_read_calls(): + self.assertEqual(call[call.index("--auth-mode") + 1], "login") + self.assertEqual(call[call.index("--container-name") + 1], "tfstate") + self.assertEqual(call[call.index("--name") + 1], "mock-tre") + + def test_missing_state_blob_is_reachable_without_initialising_state(self): + result = self.run_script(plan_exit=0, exists="false") + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(self.terraform_calls("state"), []) + self.assertEqual(len(self.blob_read_calls()), 3) + self.assertEqual([c for c in self.calls if c[:3] == ["az", "storage", "blob"]], self.blob_read_calls()) + + def test_denied_blob_access_cannot_count_as_a_missing_blob(self): + result = self.run_script(backend=(False,), backend_error="AuthorizationPermissionMismatch: HTTP 403") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.blob_read_calls()), 30) + self.assertIn("AuthorizationPermissionMismatch", self.output) + + def test_missing_or_unexpected_blob_result_cannot_pass_readiness(self): + for exists in ("", "null", "unexpected"): + with self.subTest(exists=exists): + result = self.run_script(exists=exists) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Unexpected response", self.output) + + def test_rerun_still_fails_if_dns_never_becomes_ready(self): + result = self.run_script(plan_exit=0, dns=(False,)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.terraform_calls("apply"), []) + self.assertEqual(len(self.dns_attempts()), 30) + + def test_stale_public_alias_cannot_satisfy_readiness(self): + result = self.run_script(cnames=["blob.storage.example.test."]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.dns_attempts()), 30) + self.assertEqual(self.blob_read_calls(), []) + self.assertIn("expected alias not observed", self.output) + + def test_stale_alias_resets_consecutive_successes(self): + private = "mockstorage.privatelink.blob.core.windows.net." + result = self.run_script(cnames=[private, "stale.example.test.", private, private, private]) + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(len(self.dns_attempts()), 5) + + def test_another_accounts_private_alias_is_rejected(self): + result = self.run_script(cnames=["other.privatelink.blob.core.windows.net."]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.dns_attempts()), 30) + + def test_alias_uses_actual_cloud_endpoint_and_dns_case_insensitivity(self): + result = self.run_script( + host="mockstorage.blob.core.usgovcloudapi.net", + cnames=["MOCKSTORAGE.PRIVATELINK.BLOB.CORE.USGOVCLOUDAPI.NET."], + ) + self.assertEqual(result.returncode, 0, self.output) + + def test_backend_failure_resets_and_recovers_without_logging_state(self): + result = self.run_script(backend=(True, False, True, True, True)) + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(len(self.dns_attempts()), 5) + self.assertIn("backend read failed", self.output) + self.assertNotIn("SECRET_STATE_CONTENT", self.output) + + def test_persistent_backend_failure_is_bounded_and_has_dns_diagnostics(self): + result = self.run_script(backend=(False,)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.dns_attempts()), 30) + self.assertIn("backend unavailable", self.output) + self.assertTrue(any(c[0] == "dig" and "@168.63.129.16" in c for c in self.calls)) + + def test_other_resource_processor_type_skips(self): + result = self.run_script(rp_type="other") + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(self.calls, []) + + def test_changed_target_is_applied_before_dns_checks(self): + result = self.run_script(dns=(True, True, True)) + self.assertEqual(result.returncode, 0, self.output) + plans = self.terraform_calls("plan") + self.assertEqual(len(plans), 1) + self.assertIn(f"-target={PE_ADDRESS}", plans[0]) + self.assertIn("-detailed-exitcode", plans[0]) + applies = self.terraform_calls("apply") + self.assertEqual(len(applies), 1) + self.assertEqual(applies[0][-1], plans[0][plans[0].index("-out") + 1]) + self.assertLess(self.calls.index(applies[0]), self.calls.index(self.dns_attempts()[0])) + self.assertEqual(len(self.dns_attempts()), 3) + self.assertEqual([c for c in self.calls if c[0] == "sleep"], [["sleep", "10"]] * 2) + self.assertIn(HOST, self.output) + + def test_transient_dns_failure_resets_and_recovers(self): + result = self.run_script(dns=(True, False, False, True, True, True)) + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual(len(self.dns_attempts()), 6) + self.assertIn("did not resolve", self.output) + + def test_persistent_dns_failure_is_bounded_and_reported(self): + result = self.run_script(dns=(False,)) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.dns_attempts()), 30) + self.assertEqual(len([c for c in self.calls if c[0] == "sleep"]), 29) + self.assertIn(f"{HOST} did not become ready after 30 attempts", self.output) + + def test_targeted_apply_failure_stops_before_dns_wait(self): + result = self.run_script(apply_exit=1) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.dns_attempts(), []) + self.assertIn("apply failed", self.output) + + def test_targeted_plan_failure_stops_without_applying_or_waiting(self): + result = self.run_script(plan_exit=1) + self.assertEqual(result.returncode, 1) + self.assertEqual(self.terraform_calls("apply"), []) + self.assertEqual(self.dns_attempts(), []) + self.assertIn("plan failed (exit 1)", self.output) + + +if __name__ == "__main__": + unittest.main() diff --git a/devops/tests/test_validate_ci_cleanup.py b/devops/tests/test_validate_ci_cleanup.py index 670349bc04..a7c5c42e7c 100644 --- a/devops/tests/test_validate_ci_cleanup.py +++ b/devops/tests/test_validate_ci_cleanup.py @@ -123,6 +123,7 @@ def values(name): os.execv(sys.executable, [sys.executable, *sys.argv[1:]]) sys.path.insert(0, str(Path(sys.argv[1]).resolve().parent)) import ci_cleanup_scope as scope +import ci_cleanup_activity as activity import recover_bootstrap_lease as recovery def github(ctx, suffix): @@ -137,6 +138,7 @@ def github(ctx, suffix): raise AssertionError("Unexpected GitHub request: " + suffix) scope.github = github +activity.github = github recovery.github = github sys.argv = sys.argv[1:] sys.exit(scope.main()) diff --git a/devops/tests/test_verify_deployment_lock.py b/devops/tests/test_verify_deployment_lock.py new file mode 100644 index 0000000000..bedc98dad3 --- /dev/null +++ b/devops/tests/test_verify_deployment_lock.py @@ -0,0 +1,180 @@ +"""Check delegated deployment ownership against GitHub run and queue responses.""" + +from contextlib import redirect_stderr, redirect_stdout +from copy import deepcopy +import io +import os +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "devops/scripts")) +import recover_bootstrap_lease as recovery # noqa: E402 +import verify_deployment_lock as guard # noqa: E402 + + +class DelegatedLockTests(unittest.TestCase): + def setUp(self): + self.env = patch.dict( + os.environ, + { + "CI_DEPLOYMENT_LOCK_DELEGATED": "true", + "CI_GIT_REF": "refs/heads/feature/test", + "GITHUB_ACTIONS": "true", + "GITHUB_REF": "refs/heads/feature/test", + "GITHUB_REPOSITORY": "microsoft/AzureTRE", + "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_SHA": "a" * 40, + }, + clear=True, + ) + self.env.start() + self.addCleanup(self.env.stop) + self.ctx = guard.context() + self.run = { + "id": 123, + "run_attempt": 2, + "status": "in_progress", + "event": "workflow_dispatch", + "path": ".github/workflows/deploy_tre_branch.yml", + "head_branch": "feature/test", + "head_sha": "a" * 40, + "repository": {"full_name": "microsoft/AzureTRE"}, + "head_repository": {"full_name": "microsoft/AzureTRE"}, + } + self.group = { + "group_name": "deploy-refs/heads/feature/test", + "total_count": 1, + "group_members": [{"run_id": 123, "status": "in_progress"}], + } + self.requests = [] + for module in (guard, recovery): + mock = patch.object(module, "github", side_effect=self.github) + mock.start() + self.addCleanup(mock.stop) + + def github(self, ctx, suffix): + self.assertEqual(ctx, self.ctx) + self.requests.append(suffix) + if suffix == "/actions/runs/123": + return deepcopy(self.run) + if suffix == "/actions/concurrency_groups/deploy-refs%2Fheads%2Ffeature%2Ftest": + return deepcopy(self.group) + if suffix == "/actions/jobs/456": + return { + "id": 456, + "run_id": 123, + "run_attempt": 2, + "status": "in_progress", + "name": "Deploy PR / Deploy Management", + } + self.fail(f"Unexpected GitHub request: {suffix}") + + def test_current_branch_workflow_owner_is_accepted_with_waiting_cleanup(self): + self.group["group_members"].append({"run_id": 999, "job_id": 101, "status": "pending"}) + self.group["total_count"] = 2 + guard.verify_delegated_lock(self.ctx) + self.assertEqual(len(self.requests), 2) + + def test_active_parent_run_status_uses_the_canonical_in_progress_owner(self): + for status in ("queued", "pending", "waiting", "requested"): + self.run["status"] = status + with self.subTest(status=status): + guard.verify_delegated_lock(self.ctx) + + def test_wrong_run_attempt_source_or_repository_is_refused(self): + for key, value in ( + ("id", 456), + ("id", True), + ("run_attempt", 1), + ("run_attempt", "2"), + ("status", "unknown"), + ("status", "completed"), + ("event", "issue_comment"), + ("path", ".github/workflows/pr_comment_bot.yml"), + ("head_branch", "another-branch"), + ("head_sha", "b" * 40), + ("repository", {"full_name": "another/repo"}), + ("head_repository", {"full_name": "another/repo"}), + ): + before = self.run[key] + self.run[key] = value + with self.subTest(key=key, value=value), self.assertRaises(guard.RecoveryError): + guard.verify_delegated_lock(self.ctx) + self.run[key] = before + + def test_wrong_pending_missing_or_ambiguous_owner_is_refused(self): + for members in ( + [], + [{"run_id": 999, "status": "in_progress"}], + [{"run_id": 123, "status": "pending"}], + [{"run_id": 123, "status": "in_progress"}, {"run_id": 999, "status": "in_progress"}], + ): + self.group["group_members"] = members + self.group["total_count"] = len(members) + with self.subTest(members=members), self.assertRaises(guard.RecoveryError): + guard.verify_delegated_lock(self.ctx) + + def test_management_job_lease_cannot_substitute_for_caller_workflow_lease(self): + # The existing bootstrap verifier accepts this job owner. Delegation must + # reject it because it does not protect branch preparation. + self.group["group_members"][0]["job_id"] = 456 + with self.assertRaisesRegex(guard.RecoveryError, "workflow-level"): + guard.verify_delegated_lock(self.ctx) + self.assertIn("/actions/jobs/456", self.requests) + + def test_wrong_group_or_incomplete_queue_is_refused(self): + for key, value in (("group_name", "deploy-other"), ("total_count", 2)): + before = self.group[key] + self.group[key] = value + with self.subTest(key=key), self.assertRaises(guard.RecoveryError): + guard.verify_delegated_lock(self.ctx) + self.group[key] = before + + def test_invalid_context_is_refused_before_any_request(self): + for key, value in ( + ("GITHUB_ACTIONS", "false"), + ("CI_GIT_REF", "refs/pull/123/merge"), + ("CI_GIT_REF", "refs/heads/main"), + ("CI_GIT_REF", "refs/heads/another"), + ("GITHUB_REF", "refs/heads/another"), + ("GITHUB_REPOSITORY", "../another/repo"), + ("GITHUB_RUN_ID", ""), + ("GITHUB_RUN_ATTEMPT", "0"), + ("GITHUB_SHA", "missing"), + ): + with ( + self.subTest(key=key), + patch.dict(os.environ, {key: value}), + self.assertRaises((guard.RecoveryError, ValueError)), + ): + guard.context() + self.assertEqual(self.requests, []) + + def test_other_callers_keep_the_reusable_lock_without_delegation_checks(self): + output = io.StringIO() + with patch.dict(os.environ, {"CI_DEPLOYMENT_LOCK_DELEGATED": "false"}), redirect_stdout(output): + self.assertEqual(guard.main(), 0) + self.assertEqual(self.requests, []) + self.assertIn("reusable workflow holds", output.getvalue()) + + def test_invalid_delegation_flag_and_api_failure_fail_the_command(self): + for value in ("", "yes", "TRUE"): + with patch.dict(os.environ, {"CI_DEPLOYMENT_LOCK_DELEGATED": value}), redirect_stderr(io.StringIO()): + self.assertEqual(guard.main(), 1) + output, error = io.StringIO(), io.StringIO() + with ( + patch.object(recovery, "github", side_effect=guard.RecoveryError("API unavailable")), + redirect_stdout(output), + redirect_stderr(error), + ): + self.assertEqual(guard.main(), 1) + self.assertEqual(output.getvalue(), "") + self.assertIn("API unavailable", error.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/devops/version.txt b/devops/version.txt index c98d6e334a..2e77e10734 100644 --- a/devops/version.txt +++ b/devops/version.txt @@ -1 +1 @@ -__version__ = "0.6.13" +__version__ = "0.6.14" From 2b1bf7bc00dc28ae111464ca582d4f8007b85e0a Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:40:43 +0100 Subject: [PATCH 02/17] Fix bundle execution and preserve uninstall failures (#5119) --- CHANGELOG.md | 1 + api_app/_version.py | 2 +- api_app/models/domain/operation.py | 3 + .../service_bus/deployment_status_updater.py | 82 +++- .../test_routes/test_shared_services.py | 44 ++ .../test_deployment_status_update.py | 397 ++++++++++++++++++ resource_processor/_version.py | 2 +- resource_processor/tests_rp/test_runner.py | 99 ++++- resource_processor/vmss_porter/runner.py | 27 +- .../airlock_notifier/Dockerfile.tmpl | 5 + .../airlock_notifier/porter.yaml | 4 +- .../cyclecloud/Dockerfile.tmpl | 5 + .../shared_services/cyclecloud/porter.yaml | 4 +- .../Dockerfile.tmpl | 5 + .../porter.yaml | 4 +- .../Dockerfile.tmpl | 5 + .../porter.yaml | 4 +- .../workspace_services/ohdsi/Dockerfile.tmpl | 5 + .../workspace_services/ohdsi/porter.yaml | 4 +- 19 files changed, 644 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9690aad6e9..eb11d5da17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Correct managed-identity login in bundle runtimes. Preserve failed uninstall results and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) * Wait for management storage, retry recognised state-read permission failures and validate Terraform outputs. Protect active and recently used validation environments from scheduled cleanup. ([#5252](https://github.com/microsoft/AzureTRE/pull/5252)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Validate Airlock sort fields against the supported options and avoid returning database error details to callers. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/api_app/_version.py b/api_app/_version.py index cc09a062c6..2e378084fb 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.28.3" +__version__ = "0.28.4" diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 82342315ed..2c37666a26 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -64,6 +64,9 @@ class OperationStep(AzureTREModel): status: Optional[Status] = Field(None, title="Operation step status") message: Optional[str] = Field("", title="Additional operation step status information") updatedWhen: Optional[float] = Field(default=None, title="POSIX Timestamp for When the operation step was updated") + statusUpdateComplete: bool = Field( + False, description="Whether the terminal status resource writes and next-step dispatch have completed" + ) # An example for this property will be if we have a step that is responsible for updating the firewall, and its origin was the guacamole workspace service, the id here will be the guacamole id sourceTemplateResourceId: Optional[str] = Field( default=None, title="sourceTemplateResourceId", description="Id of the parent of the resource to update" diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 299dc8a5a8..862a2c101c 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -151,7 +151,24 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise f"Error finding step {message.stepId} in operation {message.operationId}" + uninstall_pipeline = operation.action == RequestAction.UnInstall and len(operation.steps) > 1 + if uninstall_pipeline and self.ignore_uninstall_callback(operation, current_step_index, message): + if ( + not step_to_update.statusUpdateComplete + and step_to_update.status == message.status + and operation.status == Status.DeletingFailed + and any(later.is_failure() for later in operation.steps[current_step_index + 1 :]) + ): + # Dispatch can fail after the current step succeeds. Repair + # a missed root write without replaying the earlier effects. + await self.update_primary_uninstall_status(operation) + step_to_update.statusUpdateComplete = True + await self.operations_repo.update_item(operation) + return True + # update the step status + if step_to_update.status != message.status: + step_to_update.statusUpdateComplete = False step_to_update.status = message.status step_to_update.message = message.message step_to_update.updatedWhen = get_timestamp() @@ -166,11 +183,20 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_id = uuid.UUID(step_to_update.resourceId) resource = await self.resource_repo.get_resource_by_id(resource_id) - resource.deploymentStatus = step_to_update.status + resource.deploymentStatus = ( + operation.status + if uninstall_pipeline and str(resource_id) == operation.resourceId + else step_to_update.status + ) await self.resource_repo.update_item(resource) + if uninstall_pipeline and str(resource_id) != operation.resourceId: + await self.update_primary_uninstall_status(operation) # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. if not step_to_update.is_success(): + if uninstall_pipeline and step_to_update.is_failure(): + step_to_update.statusUpdateComplete = True + await self.operations_repo.update_item(operation) return True # update the resource doc to persist any outputs @@ -220,6 +246,14 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) await self.operations_repo.update_item(operation) + if uninstall_pipeline: + await self.update_primary_uninstall_status(operation) + + if uninstall_pipeline: + # Save only after the resource writes and dispatch. A retry must + # resume those effects if an earlier database write succeeded. + step_to_update.statusUpdateComplete = True + await self.operations_repo.update_item(operation) result = True @@ -232,6 +266,32 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result + def ignore_uninstall_callback(self, operation: Operation, step_index: int, message: DeploymentStatusUpdateMessage): + step = operation.steps[step_index] + if step.is_success() or step.is_failure(): + if step.status != message.status or step.statusUpdateComplete: + logger.info(f"Ignoring completed uninstall callback for {operation.id}/{step.id}") + return True + + # Dispatch follows the previous resource writes. Once a later step has + # progressed, replaying those writes could overwrite newer resource data. + awaiting = (Status.AwaitingDeployment, Status.AwaitingUpdate, Status.AwaitingDeletion, Status.AwaitingAction) + if any(later.status not in awaiting for later in operation.steps[step_index + 1 :]): + logger.info(f"Ignoring earlier uninstall callback for {operation.id}/{step.id}") + return True + + if any(not previous.is_success() for previous in operation.steps[:step_index]): + logger.info(f"Ignoring uninstall callback before its predecessors succeeded: {operation.id}/{step.id}") + return True + return False + + async def update_primary_uninstall_status(self, operation: Operation): + # Cascaded deletion can contain several main steps. The operation owns + # the root resource, which must remain visible until every step finishes. + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(operation.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + async def update_overall_operation_status(self, operation: Operation, step: OperationStep, is_last_step: bool): operation.updatedWhen = get_timestamp() @@ -248,17 +308,15 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.status = self.get_failure_status_for_action(operation.action) operation.message = f"Multi step pipeline failed on step {step.templateStepId}" - # pipeline failed - update the primary resource (from the main step) as failed too - main_step = None - for i, step in enumerate(operation.steps): - if step.templateStepId == "main": - main_step = step - break - - if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) + # Uninstall pipelines persist the root status in the caller. Keep + # the existing main-step behaviour for other pipeline actions. + if operation.action != RequestAction.UnInstall: + main_step = next((item for item in operation.steps if item.templateStepId == "main"), None) + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) + return if step.is_success() and is_last_step: operation.status = self.get_success_status_for_action(operation.action) diff --git a/api_app/tests_ma/test_api/test_routes/test_shared_services.py b/api_app/tests_ma/test_api/test_routes/test_shared_services.py index f168a46416..e4fe04d5a2 100644 --- a/api_app/tests_ma/test_api/test_routes/test_shared_services.py +++ b/api_app/tests_ma/test_api/test_routes/test_shared_services.py @@ -1,4 +1,6 @@ import random +import sqlite3 +from contextlib import closing from unittest.mock import AsyncMock import uuid import pytest @@ -12,6 +14,8 @@ from db.errors import EntityDoesNotExist from models.domain.shared_service import SharedService +from models.domain.operation import Status +from db.repositories.shared_services import SharedServiceRepository from resources import strings from auth.rbac import require_tre_admin, require_tre_user_or_admin from azure.cosmos.exceptions import CosmosAccessConditionFailedError @@ -478,3 +482,43 @@ async def test_patch_shared_service_with_invalid_field_returns_422(self, _, app, # Check that the error message contains the key information about the validation error assert "fakeField" in response.text assert "extra inputs are not permitted" in response.text.lower() + + @pytest.mark.parametrize( + "resource_status, expected_http_status", + [(Status.PipelineRunning, 200), (Status.DeletingFailed, 200), (Status.Deleted, 404)], + ) + async def test_uninstall_operation_remains_readable_until_pipeline_finishes( + self, app, client, resource_status, expected_http_status + ): + resource = sample_shared_service() + resource.isEnabled = False + resource.deploymentStatus = resource_status + operation = sample_resource_operation(resource_id=SHARED_SERVICE_ID, operation_id=OPERATION_ID) + operation.action = "uninstall" + operation.status = resource_status + + # Execute the repository's actual equality predicates locally. Only the + # database transport is replaced; the lookup and HTTP dependencies run. + with closing(sqlite3.connect(":memory:")) as database: + database.execute("CREATE TABLE c (id TEXT, resourceType TEXT, deploymentStatus TEXT)") + database.execute("INSERT INTO c VALUES (?, ?, ?)", (resource.id, resource.resourceType, resource_status)) + + def query_documents(query, parameters): + rows = database.execute(query, {item["name"].removeprefix("@"): item["value"] for item in parameters}) + return [resource.model_dump() for _ in rows] + + with ( + patch.object(SharedServiceRepository, "query", side_effect=query_documents), + patch( + "api.dependencies.operations.OperationRepository.get_operation_by_id_for_resource", + return_value=operation, + ) as operation_lookup, + ): + response = await client.get(f"/api/shared-services/{SHARED_SERVICE_ID}/operations/{OPERATION_ID}") + + assert response.status_code == expected_http_status + if expected_http_status == 200: + assert response.json()["operation"]["status"] == resource_status + operation_lookup.assert_awaited_once_with(operation_id=OPERATION_ID, resource_id=SHARED_SERVICE_ID) + else: + operation_lookup.assert_not_awaited() diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index fd215ffa86..a0459cff70 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -11,6 +11,7 @@ from db.errors import EntityDoesNotExist from models.domain.workspace import Workspace +from models.domain.workspace_service import WorkspaceService from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status from resources import strings from service_bus.deployment_status_updater import DeploymentStatusUpdater @@ -510,6 +511,90 @@ async def test_multi_step_operation_ends_at_last_step( sb_sender_client().get_queue_sender().send_messages.assert_not_called() +@patch("service_bus.deployment_status_updater.ResourceHistoryRepository.create") +@patch("service_bus.deployment_status_updater.ResourceTemplateRepository.create") +@patch("service_bus.deployment_status_updater.update_resource_for_step") +@patch("service_bus.deployment_status_updater.OperationRepository.create") +@patch("service_bus.deployment_status_updater.ResourceRepository.create") +@patch("service_bus.helpers.ServiceBusClient") +@pytest.mark.parametrize("child_status", [Status.Deleted, Status.DeletingFailed]) +async def test_child_uninstall_controls_parent_deletion( + sb_sender_client, + resource_repo, + operations_repo, + update_resource_for_step, + _, + __, + multi_step_operation, + user_resource_multi, + primary_workspace_service_resource, + child_status, +): + operation = copy.deepcopy(multi_step_operation) + operation.id = OPERATION_ID + user_resource_multi.id = "0e569b65-7e01-4a0b-a971-a41f5110c0c1" + primary_workspace_service_resource.id = "dc9b2f53-0784-4c4f-9b73-4a060a385bca" + parent = WorkspaceService(**primary_workspace_service_resource.model_dump(), workspaceId="test-workspace") + parent.isEnabled = False + parent.deploymentStatus = Status.Updated + operation.action = RequestAction.UnInstall + operation.steps = operation.steps[:2] + child_step, parent_step = operation.steps + child_step.resourceId = user_resource_multi.id + child_step.resourceAction = RequestAction.UnInstall + child_step.status = Status.Deleting + parent_step.resourceId = primary_workspace_service_resource.id + parent_step.resourceAction = RequestAction.UnInstall + parent_step.status = Status.AwaitingDeletion + operation.resourceId = primary_workspace_service_resource.id + operation.status = Status.PipelineRunning + for step, resource in [(child_step, user_resource_multi), (parent_step, parent)]: + step.templateStepId = "main" + step.sourceTemplateResourceId = resource.id + step.resourceTemplateName = resource.templateName + step.resourceType = resource.resourceType + parent_before = parent_step.model_copy(deep=True) + + message = { + "operationId": operation.id, + "stepId": child_step.id, + "id": child_step.resourceId, + "status": child_status, + "message": "Child uninstall result", + } + operations_repo.return_value.get_operation_by_id.return_value = operation + resources = { + user_resource_multi.id: user_resource_multi, + parent.id: parent, + } + resource_repo.return_value.get_resource_by_id.side_effect = lambda resource_id: resources[str(resource_id)] + resource_repo.return_value.get_resource_dict_by_id.side_effect = lambda resource_id: resources[ + str(resource_id) + ].model_dump() + update_resource_for_step.return_value = parent + sb_sender_client().get_queue_sender().send_messages = AsyncMock() + + updater = DeploymentStatusUpdater() + await updater.init_repos() + complete = await updater.process_message(ServiceBusReceivedMessageMock(message)) + + assert complete is True + saved_operation = operations_repo.return_value.update_item.call_args.args[0] + expected_parent_status = Status.DeletingFailed if child_status == Status.DeletingFailed else Status.PipelineRunning + assert saved_operation.status == expected_parent_status + assert saved_operation.steps[0].status == child_status + assert saved_operation.steps[0].message == message["message"] + assert saved_operation.steps[1] == parent_before + assert user_resource_multi.deploymentStatus == child_status + assert parent.deploymentStatus == expected_parent_status + if child_status == Status.DeletingFailed: + update_resource_for_step.assert_not_called() + sb_sender_client().get_queue_sender().send_messages.assert_not_called() + else: + update_resource_for_step.assert_called_once() + sb_sender_client().get_queue_sender().send_messages.assert_called_once() + + async def test_convert_outputs_to_dict(): # Test case 1: Empty list of outputs outputs_list = [] @@ -533,3 +618,315 @@ async def test_convert_outputs_to_dict(): "list2": ["one", "two"], } assert status_updater.convert_outputs_to_dict(deployment_status_update_message.outputs) == expected_result + + +@pytest.mark.parametrize( + "phase, message_status, expected_operation_status, expected_primary_status", + [ + ("main", Status.Deleted, Status.PipelineRunning, Status.PipelineRunning), + ("final", Status.Updated, Status.Deleted, Status.Deleted), + ("final", Status.UpdatingFailed, Status.DeletingFailed, Status.DeletingFailed), + ("enqueue_failed", Status.Deleted, Status.DeletingFailed, Status.DeletingFailed), + ], +) +async def test_uninstall_waits_for_final_pipeline_step( + phase, message_status, expected_operation_status, expected_primary_status +): + primary_id = "d9236f8f-7fbb-46d3-9c94-dea6daf32b7f" + dependency_id = "7af8f146-d751-4587-a22e-eb44c1c6fbe2" + primary = create_sample_workspace_object(primary_id) + primary.isEnabled = False + primary.deploymentStatus = Status.PipelineRunning if phase == "final" else Status.Deleting + dependency = create_sample_workspace_object(dependency_id) + dependency.deploymentStatus = Status.Updating + stored = {primary_id: primary, dependency_id: dependency} + + operation = create_sample_operation(primary_id, RequestAction.UnInstall) + operation.steps[0].status = Status.Deleted if phase == "final" else Status.Deleting + operation.steps.append( + OperationStep( + id="firewall-step", + templateStepId="remove-firewall-rules", + resourceId=dependency_id, + resourceAction=RequestAction.Upgrade, + sourceTemplateResourceId=primary_id, + status=Status.Updating if phase == "final" else Status.AwaitingUpdate, + ) + ) + operation.status = Status.PipelineRunning + current_step = operation.steps[1 if phase == "final" else 0] + message = DeploymentStatusUpdateMessage( + operationId=operation.id, + stepId=current_step.id, + id=current_step.resourceId, + status=message_status, + message="Final firewall update failed" if message_status == Status.UpdatingFailed else "Completed", + ) + + updater = DeploymentStatusUpdater() + updater.operations_repo = AsyncMock() + updater.operations_repo.get_operation_by_id.return_value = operation + updater.resource_repo = AsyncMock() + updater.resource_template_repo = AsyncMock() + updater.resource_history_repo = AsyncMock() + updater.resource_repo.get_resource_by_id.side_effect = lambda resource_id: stored[str(resource_id)].model_copy( + deep=True + ) + updater.resource_repo.get_resource_dict_by_id.side_effect = lambda resource_id: stored[ + str(resource_id) + ].model_dump() + + def save_resource(resource): + stored[resource.id] = resource.model_copy(deep=True) + + def save_resource_dict(resource): + stored[resource["id"]] = stored[resource["id"]].model_copy(update=resource, deep=True) + + updater.resource_repo.update_item.side_effect = save_resource + updater.resource_repo.update_item_dict.side_effect = save_resource_dict + with ( + patch("service_bus.deployment_status_updater.update_resource_for_step", return_value=dependency), + patch("service_bus.deployment_status_updater.send_deployment_message", new_callable=AsyncMock) as send, + ): + if phase == "enqueue_failed": + send.side_effect = RuntimeError("Cannot enqueue final step") + assert await updater.update_status_in_database(message) is True + + assert operation.status == expected_operation_status + assert stored[primary_id].deploymentStatus == expected_primary_status + assert stored[primary_id].isEnabled is False + if phase != "final": + assert stored[primary_id].deploymentStatus != Status.Deleted + send.assert_awaited_once() + else: + assert stored[dependency_id].deploymentStatus == message_status + send.assert_not_awaited() + if message_status == Status.UpdatingFailed: + assert operation.message == "Multi step pipeline failed on step remove-firewall-rules" + assert operation.steps[-1].message == "Final firewall update failed" + if phase == "enqueue_failed": + assert operation.steps[-1].status == Status.UpdatingFailed + assert "Cannot enqueue final step" in operation.steps[-1].message + + +@pytest.fixture +def persisted_uninstall_pipeline(): + primary = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") + primary.isEnabled = False + primary.deploymentStatus = Status.Deleting + dependency = create_sample_workspace_object("7af8f146-d751-4587-a22e-eb44c1c6fbe2") + dependency.deploymentStatus = Status.Updated + operation = create_sample_operation(primary.id, RequestAction.UnInstall) + operation.status = Status.PipelineRunning + operation.steps[0].status = Status.Deleting + operation.steps.append( + OperationStep( + id="firewall-step", + templateStepId="remove-firewall-rules", + resourceId=dependency.id, + resourceAction=RequestAction.Upgrade, + sourceTemplateResourceId=primary.id, + status=Status.AwaitingUpdate, + ) + ) + stored = {"operation": operation, "resources": {primary.id: primary, dependency.id: dependency}} + updater = DeploymentStatusUpdater() + updater.operations_repo = AsyncMock() + updater.resource_repo = AsyncMock() + updater.resource_template_repo = AsyncMock() + updater.resource_history_repo = AsyncMock() + updater.operations_repo.get_operation_by_id.side_effect = lambda _: stored["operation"].model_copy(deep=True) + + def save_operation(item): + stored["operation"] = item.model_copy(deep=True) + + def save_resource(item): + stored["resources"][item.id] = item.model_copy(deep=True) + + def save_resource_dict(item): + original = stored["resources"][item["id"]] + stored["resources"][item["id"]] = original.model_copy(update=item, deep=True) + + updater.operations_repo.update_item.side_effect = save_operation + updater.resource_repo.get_resource_by_id.side_effect = lambda item_id: stored["resources"][str(item_id)].model_copy( + deep=True + ) + updater.resource_repo.get_resource_dict_by_id.side_effect = lambda item_id: stored["resources"][ + str(item_id) + ].model_dump() + updater.resource_repo.update_item.side_effect = save_resource + updater.resource_repo.update_item_dict.side_effect = save_resource_dict + + def message(step_index, status): + step = stored["operation"].steps[step_index] + return DeploymentStatusUpdateMessage( + operationId=operation.id, stepId=step.id, id=step.resourceId, status=status, message="Result" + ) + + with ( + patch("service_bus.deployment_status_updater.update_resource_for_step", return_value=dependency), + patch("service_bus.deployment_status_updater.send_deployment_message", new_callable=AsyncMock) as send, + ): + yield updater, stored, message, send + + +async def test_uninstall_redelivery_does_not_dispatch_next_step_twice(persisted_uninstall_pipeline): + updater, stored, message, send = persisted_uninstall_pipeline + callback = message(0, Status.Deleted) + assert await updater.update_status_in_database(callback) is True + assert stored["operation"].steps[0].statusUpdateComplete is True + before = copy.deepcopy(stored) + + assert await updater.update_status_in_database(callback) is True + + assert stored == before + send.assert_awaited_once() + + +@pytest.mark.parametrize("final_status", [Status.Updated, Status.UpdatingFailed]) +@pytest.mark.parametrize( + "callback_index, callback_status", + [(0, Status.Deleted), (0, Status.Deleting), (1, Status.Updating), (1, Status.Updated)], +) +async def test_uninstall_redelivery_preserves_final_outcome( + persisted_uninstall_pipeline, final_status, callback_index, callback_status +): + updater, stored, message, send = persisted_uninstall_pipeline + assert await updater.update_status_in_database(message(0, Status.Deleted)) is True + assert await updater.update_status_in_database(message(1, final_status)) is True + before = copy.deepcopy(stored) + + assert await updater.update_status_in_database(message(callback_index, callback_status)) is True + + assert stored == before + send.assert_awaited_once() + + +@pytest.mark.parametrize("later_status", [Status.Updating, Status.Updated, Status.UpdatingFailed]) +async def test_uninstall_older_callback_does_not_repeat_writes_after_later_progress( + persisted_uninstall_pipeline, later_status +): + updater, stored, message, send = persisted_uninstall_pipeline + # Older operations have no checkpoint. A later callback also proves dispatch + # completed if the process stopped before it could save the checkpoint. + operation = stored["operation"] + operation.steps[0].status = Status.Deleted + operation.steps[1].status = later_status + operation.status = { + Status.Updating: Status.PipelineRunning, + Status.Updated: Status.Deleted, + Status.UpdatingFailed: Status.DeletingFailed, + }[later_status] + legacy = operation.model_dump() + for step in legacy["steps"]: + del step["statusUpdateComplete"] + stored["operation"] = Operation.model_validate(legacy) + before = copy.deepcopy(stored) + + assert await updater.update_status_in_database(message(0, Status.Deleted)) is True + + if later_status == Status.UpdatingFailed: + # A legacy record can also be an interrupted dispatch-failure write. + # Only the saved root outcome and the earlier checkpoint need repair. + before["operation"].steps[0].statusUpdateComplete = True + before["resources"][operation.resourceId].deploymentStatus = Status.DeletingFailed + updater.resource_repo.update_item.assert_awaited_once_with(before["resources"][operation.resourceId]) + else: + updater.resource_repo.update_item.assert_not_awaited() + assert stored == before + updater.resource_repo.update_item_dict.assert_not_awaited() + send.assert_not_awaited() + + +@pytest.mark.parametrize("write_method", ["update_item", "update_item_dict"]) +async def test_uninstall_redelivery_resumes_after_partial_resource_write(persisted_uninstall_pipeline, write_method): + updater, stored, message, send = persisted_uninstall_pipeline + writer = getattr(updater.resource_repo, write_method) + save = writer.side_effect + writer.side_effect = RuntimeError("Interrupted database write") + callback = message(0, Status.Deleted) + assert await updater.update_status_in_database(callback) is False + assert stored["operation"].steps[0].status == Status.Deleted + assert stored["operation"].steps[0].statusUpdateComplete is False + send.assert_not_awaited() + + writer.side_effect = save + assert await updater.update_status_in_database(callback) is True + + assert stored["operation"].steps[0].statusUpdateComplete is True + assert stored["resources"][str(callback.id)].deploymentStatus == Status.PipelineRunning + send.assert_awaited_once() + + +@pytest.mark.parametrize("final_status", [Status.Updated, Status.UpdatingFailed]) +async def test_uninstall_redelivery_finishes_terminal_resource_write(persisted_uninstall_pipeline, final_status): + updater, stored, message, send = persisted_uninstall_pipeline + assert await updater.update_status_in_database(message(0, Status.Deleted)) is True + writer = updater.resource_repo.update_item + save = writer.side_effect + + def fail_primary_write(item): + if item.id == stored["operation"].resourceId: + raise RuntimeError("Interrupted primary resource write") + save(item) + + writer.side_effect = fail_primary_write + callback = message(1, final_status) + assert await updater.update_status_in_database(callback) is False + assert stored["operation"].steps[1].statusUpdateComplete is False + + writer.side_effect = save + assert await updater.update_status_in_database(callback) is True + + expected = Status.Deleted if final_status == Status.Updated else Status.DeletingFailed + assert stored["operation"].status == expected + assert stored["operation"].steps[1].statusUpdateComplete is True + assert stored["resources"][stored["operation"].resourceId].deploymentStatus == expected + send.assert_awaited_once() + + +async def test_uninstall_checkpoint_failure_does_not_report_dispatch_failure(persisted_uninstall_pipeline): + updater, stored, message, send = persisted_uninstall_pipeline + writer = updater.operations_repo.update_item + save = writer.side_effect + + def fail_checkpoint(item): + if item.steps[0].statusUpdateComplete: + raise RuntimeError("Interrupted checkpoint write") + save(item) + + writer.side_effect = fail_checkpoint + assert await updater.update_status_in_database(message(0, Status.Deleted)) is False + + assert stored["operation"].status == Status.PipelineRunning + assert stored["operation"].steps[1].status == Status.AwaitingUpdate + assert stored["operation"].steps[0].statusUpdateComplete is False + send.assert_awaited_once() + + +async def test_uninstall_redelivery_repairs_root_after_dispatch_failure(persisted_uninstall_pipeline): + updater, stored, message, send = persisted_uninstall_pipeline + send.side_effect = RuntimeError("Cannot enqueue final step") + writer = updater.resource_repo.update_item + save = writer.side_effect + + def fail_failure_write(item): + if item.deploymentStatus == Status.DeletingFailed: + raise RuntimeError("Interrupted root failure write") + save(item) + + writer.side_effect = fail_failure_write + callback = message(0, Status.Deleted) + assert await updater.update_status_in_database(callback) is False + assert stored["operation"].status == Status.DeletingFailed + assert stored["resources"][str(callback.id)].deploymentStatus == Status.PipelineRunning + assert stored["operation"].steps[0].statusUpdateComplete is False + updater.resource_repo.update_item_dict.reset_mock() + + writer.side_effect = save + assert await updater.update_status_in_database(callback) is True + + assert stored["resources"][str(callback.id)].deploymentStatus == Status.DeletingFailed + assert stored["operation"].steps[0].statusUpdateComplete is True + updater.resource_repo.update_item_dict.assert_not_awaited() + send.assert_awaited_once() diff --git a/resource_processor/_version.py b/resource_processor/_version.py index 17b059f281..a7d2b081e1 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.13.12" +__version__ = "0.13.13" diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 974044b782..7962a18972 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -492,24 +492,48 @@ async def test_invoke_porter_action_upgrade_failure_install_failure( @pytest.mark.asyncio -@patch("vmss_porter.runner.build_porter_command", return_value=([["porter", "install"]], None, "tre-params-test", None)) -@patch("vmss_porter.runner.run_porter", return_value=(1, "", "could not find installation")) -@patch("vmss_porter.runner.service_bus_message_generator", return_value="test_message") -async def test_invoke_porter_action_uninstall_failure( - mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client -): - """Test invoking a porter action with uninstall failure.""" - mock_sb_client = AsyncMock(spec=ServiceBusClient) - mock_sb_sender = AsyncMock() - mock_sb_client.get_queue_sender.return_value = mock_sb_sender - +@pytest.mark.parametrize( + "action, expected_status", + [ + ("install", "deployment_failed"), + ("upgrade", "updating_failed"), + ("uninstall", "deleting_failed"), + ("custom-action", "action_failed"), + ], +) +@pytest.mark.parametrize( + "error", + [ + ( + "could not find installation /test_id: could not read storage schema document: " + + "rpc error: code = DeadlineExceeded desc = server selection error: context deadline exceeded" + ), + "could not find installation /test_id: installation does not exist", + "", + None, + ], + ids=["storage-timeout", "missing-installation", "empty-error", "no-error"], +) +async def test_invoke_porter_action_nonzero_exit_reports_failure(action, expected_status, error): + sender = AsyncMock() + client = Mock() + client.get_queue_sender.return_value = sender config = {"deployment_status_queue": "test_queue"} - msg_body = {"id": "test_id", "action": "uninstall", "stepId": "test_step_id", "operationId": "test_operation_id"} + msg_body = {"id": "test_id", "action": action, "stepId": "test_step_id", "operationId": "test_operation_id"} - result = await invoke_porter_action(msg_body, mock_sb_client, config) + with ( + patch("vmss_porter.runner.build_porter_command", return_value=([["porter", action]], None, "params", None)), + patch("vmss_porter.runner.run_porter", return_value=(1, "", error)), + patch("vmss_porter.runner.get_porter_outputs", return_value=(True, {})) as outputs, + ): + result = await invoke_porter_action(msg_body, client, config) - assert result is True - mock_sb_sender.send_messages.assert_called() + assert result is False + outputs.assert_not_awaited() + assert sender.send_messages.await_count == 2 + final_status = json.loads(str(sender.send_messages.call_args.args[0])) + assert final_status["status"] == expected_status + assert (error or "Porter exited with code 1 and no error output.") in final_status["message"] @pytest.mark.asyncio @@ -788,6 +812,51 @@ async def test_invoke_porter_action_reports_legacy_guard_and_cleans_up(tmp_path) assert "Upgrade this resource" in final_status["message"] +@pytest.mark.asyncio +async def test_uninstall_storage_failure_after_parameter_check_preserves_failure(tmp_path): + parameters = tmp_path / "parameters.json" + installation = tmp_path / "installation.json" + parameters.write_text("parameter references") + installation.write_text("installation document") + action = ["porter", "uninstall", "resource"] + config = { + "azure_environment": "AzureCloud", + "vmss_msi_id": "msi", + "registry_server": "registry.azurecr.io", + "deployment_status_queue": "status", + } + msg_body = {"id": "resource", "action": "uninstall", "operationId": "operation", "stepId": "step"} + sender = AsyncMock() + client = Mock() + client.get_queue_sender.return_value = sender + error = "could not find installation /resource: could not read storage schema document: DeadlineExceeded" + responses = [(0, None, None)] * 5 + [ + (0, '[{"name":"resource"}]', None), + (0, '{"name":"resource","status":{"runId":"old-run"},"parameters":{}}', None), + (1, None, error), + (0, None, None), + ] + with ( + patch( + "vmss_porter.runner.build_porter_command", + return_value=([action], str(parameters), "test-parameters", str(installation)), + ), + patch("vmss_porter.runner.run_command_helper", new_callable=AsyncMock, side_effect=responses) as command, + patch("vmss_porter.runner.get_porter_outputs") as outputs, + ): + result = await invoke_porter_action(msg_body, client, config) + + assert result is False + assert not parameters.exists() + assert not installation.exists() + assert command.await_args_list[-2].args[0] == action + assert command.await_args_list[-1].args[0] == ["porter", "parameters", "delete", "test-parameters"] + outputs.assert_not_awaited() + final_status = json.loads(str(sender.send_messages.call_args.args[0])) + assert final_status["status"] == "deleting_failed" + assert error in final_status["message"] + + @pytest.mark.asyncio @patch("vmss_porter.runner.run_command_helper") async def test_run_porter_azure_login_failure(mock_run_command_helper): diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 3d731fd9bb..38e2327066 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -377,33 +377,22 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf logger.debug(f"Best-effort cleanup failed for '{installation_id}': {e}") logger.debug("Finished running porter execution command.") - action_completed_without_error = False - - if returncode == 0: - action_completed_without_error = True + action_completed_without_error = returncode == 0 # Handle command output - if returncode != 0 and err is not None: + if returncode != 0: # Construct a readable representation of the command(s) command_representation = "" for cmd in porter_command: command_representation += " ".join(cmd) + "; " command_representation = command_representation.rstrip("; ") - error_message = "Error message: " + " ".join(err.split("\n")) + "; Porter command: " + command_representation - action_completed_without_error = False - - if "uninstall" == action and "could not find installation" in err: - logger.warning( - "The installation doesn't exist. Treating as a successful action to allow the flow to proceed." - ) - action_completed_without_error = True - error_message = f"A success despite of underlying error. {error_message}" - - if action_completed_without_error: - status_for_sb_message = statuses.pass_status_string_for[action] - else: - status_for_sb_message = statuses.failed_status_string_for[action] + error_output = err or f"Porter exited with code {returncode} and no error output." + error_message = ( + "Error message: " + " ".join(error_output.split("\n")) + "; Porter command: " + command_representation + ) + # Installation lookup errors can wrap storage failures. They do not prove that resources were deleted. + status_for_sb_message = statuses.failed_status_string_for[action] resource_request_message = service_bus_message_generator(msg_body, status_for_sb_message, error_message) diff --git a/templates/shared_services/airlock_notifier/Dockerfile.tmpl b/templates/shared_services/airlock_notifier/Dockerfile.tmpl index 816a6fb388..a06203172f 100644 --- a/templates/shared_services/airlock_notifier/Dockerfile.tmpl +++ b/templates/shared_services/airlock_notifier/Dockerfile.tmpl @@ -14,6 +14,11 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ # PORTER_MIXINS +# Fail the build if the bundled CLI cannot select a user-assigned identity. +RUN az login --help > /tmp/azure-login-help && \ + grep -q -- '--client-id' /tmp/azure-login-help && \ + rm /tmp/azure-login-help + # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/shared_services/airlock_notifier/porter.yaml b/templates/shared_services/airlock_notifier/porter.yaml index 87274084b3..40e1a583de 100644 --- a/templates/shared_services/airlock_notifier/porter.yaml +++ b/templates/shared_services/airlock_notifier/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-airlock-notifier -version: 1.0.13 +version: 1.0.14 description: "A shared service notifying on Airlock Operations" registry: azuretre dockerfile: Dockerfile.tmpl @@ -75,7 +75,7 @@ parameters: mixins: - exec - az: - clientVersion: 2.67.0 + clientVersion: 2.81.0 - terraform: clientVersion: 1.9.8 diff --git a/templates/shared_services/cyclecloud/Dockerfile.tmpl b/templates/shared_services/cyclecloud/Dockerfile.tmpl index abc241c3a1..1b9c510f9a 100644 --- a/templates/shared_services/cyclecloud/Dockerfile.tmpl +++ b/templates/shared_services/cyclecloud/Dockerfile.tmpl @@ -11,5 +11,10 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ # PORTER_MIXINS +# Fail the build if the bundled CLI cannot select a user-assigned identity. +RUN az login --help > /tmp/azure-login-help && \ + grep -q -- '--client-id' /tmp/azure-login-help && \ + rm /tmp/azure-login-help + # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/shared_services/cyclecloud/porter.yaml b/templates/shared_services/cyclecloud/porter.yaml index a65f7d24ec..1cd58af1d4 100644 --- a/templates/shared_services/cyclecloud/porter.yaml +++ b/templates/shared_services/cyclecloud/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-cyclecloud -version: 0.7.8 +version: 0.7.9 description: "An Azure TRE Shared Service Template for Azure Cyclecloud" registry: azuretre dockerfile: Dockerfile.tmpl @@ -71,7 +71,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.67.0 + clientVersion: 2.81.0 install: - terraform: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl index 4a0d5cc05c..d824805e29 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl @@ -29,6 +29,11 @@ COPY --from=import-reviewvm ./terraform/download_review_data.ps1 ${BUNDLE_DIR}/t # PORTER_MIXINS +# Fail the build if the bundled CLI cannot select a user-assigned identity. +RUN az login --help > /tmp/azure-login-help && \ + grep -q -- '--client-id' /tmp/azure-login-help && \ + rm /tmp/azure-login-help + # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml index 632094c4ac..742091dd3f 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-export-reviewvm -version: 2.0.6 +version: 2.0.7 description: "An Azure TRE User Resource Template for reviewing Airlock export requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -122,7 +122,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.67.0 + clientVersion: 2.81.0 install: - terraform: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl index c77706ca7a..2850da0648 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl @@ -14,6 +14,11 @@ COPY --from=windowsvm ./terraform/vm ${BUNDLE_DIR}/terraform/vm # PORTER_MIXINS +# Fail the build if the bundled CLI cannot select a user-assigned identity. +RUN az login --help > /tmp/azure-login-help && \ + grep -q -- '--client-id' /tmp/azure-login-help && \ + rm /tmp/azure-login-help + # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml index 1e06fcaca2..8bacaa7ae9 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-import-reviewvm -version: 2.0.3 +version: 2.0.4 description: "An Azure TRE User Resource Template for reviewing Airlock import requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -129,7 +129,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.67.0 + clientVersion: 2.81.0 install: - terraform: diff --git a/templates/workspace_services/ohdsi/Dockerfile.tmpl b/templates/workspace_services/ohdsi/Dockerfile.tmpl index b2674f6d8a..5a9591d773 100644 --- a/templates/workspace_services/ohdsi/Dockerfile.tmpl +++ b/templates/workspace_services/ohdsi/Dockerfile.tmpl @@ -33,5 +33,10 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ # PORTER_MIXINS +# Fail the build if the bundled CLI cannot select a user-assigned identity. +RUN az login --help > /tmp/azure-login-help && \ + grep -q -- '--client-id' /tmp/azure-login-help && \ + rm /tmp/azure-login-help + # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index de2e88aba2..dba38adcd1 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.12 +version: 0.3.13 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl @@ -70,7 +70,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.67.0 + clientVersion: 2.81.0 - exec outputs: From 8c834dc6d720af7ec65fadf4771146d5918fa122 Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 13:58:55 +0100 Subject: [PATCH 03/17] Allow uninstall cleanup after confirmed Porter absence (#5228) --- CHANGELOG.md | 2 +- resource_processor/_version.py | 2 +- resource_processor/tests_rp/test_runner.py | 113 ++++++++++++++++++++- resource_processor/vmss_porter/runner.py | 43 +++++++- 4 files changed, 154 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb11d5da17..c5c33bf643 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: -* Correct managed-identity login in bundle runtimes. Preserve failed uninstall results and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) +* Correct managed-identity login in bundle runtimes. Preserve failed uninstall results, allow cleanup when Porter confirms that an installation is absent, and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) * Wait for management storage, retry recognised state-read permission failures and validate Terraform outputs. Protect active and recently used validation environments from scheduled cleanup. ([#5252](https://github.com/microsoft/AzureTRE/pull/5252)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Validate Airlock sort fields against the supported options and avoid returning database error details to callers. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/resource_processor/_version.py b/resource_processor/_version.py index a7d2b081e1..23a4e3cb02 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.13.13" +__version__ = "0.13.14" diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 7962a18972..857204caa1 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -514,7 +514,9 @@ async def test_invoke_porter_action_upgrade_failure_install_failure( ], ids=["storage-timeout", "missing-installation", "empty-error", "no-error"], ) -async def test_invoke_porter_action_nonzero_exit_reports_failure(action, expected_status, error): +async def test_invoke_porter_action_nonzero_exit_without_confirmed_absence_reports_failure( + action, expected_status, error +): sender = AsyncMock() client = Mock() client.get_queue_sender.return_value = sender @@ -524,6 +526,7 @@ async def test_invoke_porter_action_nonzero_exit_reports_failure(action, expecte with ( patch("vmss_porter.runner.build_porter_command", return_value=([["porter", action]], None, "params", None)), patch("vmss_porter.runner.run_porter", return_value=(1, "", error)), + patch("vmss_porter.runner.run_command_helper", return_value=(1, None, "Storage is unavailable")) as lookup, patch("vmss_porter.runner.get_porter_outputs", return_value=(True, {})) as outputs, ): result = await invoke_porter_action(msg_body, client, config) @@ -534,6 +537,10 @@ async def test_invoke_porter_action_nonzero_exit_reports_failure(action, expecte final_status = json.loads(str(sender.send_messages.call_args.args[0])) assert final_status["status"] == expected_status assert (error or "Porter exited with code 1 and no error output.") in final_status["message"] + if action == "uninstall" and "could not find installation" in (error or ""): + lookup.assert_awaited_once() + else: + lookup.assert_not_awaited() @pytest.mark.asyncio @@ -835,6 +842,7 @@ async def test_uninstall_storage_failure_after_parameter_check_preserves_failure (0, '{"name":"resource","status":{"runId":"old-run"},"parameters":{}}', None), (1, None, error), (0, None, None), + (1, None, "Storage is still unavailable"), ] with ( patch( @@ -849,14 +857,113 @@ async def test_uninstall_storage_failure_after_parameter_check_preserves_failure assert result is False assert not parameters.exists() assert not installation.exists() - assert command.await_args_list[-2].args[0] == action - assert command.await_args_list[-1].args[0] == ["porter", "parameters", "delete", "test-parameters"] + assert command.await_args_list[-3].args[0] == action + assert command.await_args_list[-2].args[0] == ["porter", "parameters", "delete", "test-parameters"] + assert command.await_args_list[-1].args[0] == [ + "porter", + "installations", + "list", + "--name", + "resource", + "--output", + "json", + "--verbosity", + "error", + ] outputs.assert_not_awaited() final_status = json.loads(str(sender.send_messages.call_args.args[0])) assert final_status["status"] == "deleting_failed" assert error in final_status["message"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "confirmation, expected_success", + [ + pytest.param((0, "[]", None), True, id="confirmed-absent"), + pytest.param((0, '[{"name":"resource"}]', None), False, id="installation-exists"), + pytest.param((0, '[{"name":"another-resource"}]', None), False, id="unexpected-installation"), + pytest.param((1, "[]", "Storage is unavailable"), False, id="failed-empty-lookup"), + pytest.param((0, "invalid JSON", None), False, id="invalid-json"), + pytest.param((0, "{}", None), False, id="object-response"), + pytest.param((0, "null", None), False, id="null-response"), + pytest.param((0, "", None), False, id="empty-output"), + pytest.param((0, None, None), False, id="no-output"), + pytest.param(RuntimeError("Lookup failed"), False, id="lookup-exception"), + pytest.param(TimeoutError(), False, id="lookup-timeout"), + pytest.param(asyncio.CancelledError(), None, id="cancelled-lookup"), + ], +) +async def test_uninstall_missing_installation_requires_fresh_confirmation(tmp_path, confirmation, expected_success): + parameters = tmp_path / "parameters.json" + installation = tmp_path / "installation.json" + parameters.write_text("parameter references") + installation.write_text("installation document") + action = ["porter", "uninstall", "resource"] + config = { + "azure_environment": "AzureCloud", + "vmss_msi_id": "msi", + "registry_server": "registry.azurecr.io", + "deployment_status_queue": "status", + } + message = {"id": "resource", "action": "uninstall", "operationId": "operation", "stepId": "step"} + sender = AsyncMock() + client = Mock() + client.get_queue_sender.return_value = sender + error = "could not find installation /resource: installation does not exist" + # The parameter check already saw an empty list. It cannot replace the fresh + # lookup after uninstall fails, because storage availability can change. + responses = [(0, None, None)] * 5 + [ + (0, "[]", None), + (1, None, error), + (0, None, None), + confirmation, + ] + with ( + patch( + "vmss_porter.runner.build_porter_command", + return_value=([action], str(parameters), "test-parameters", str(installation)), + ), + patch("vmss_porter.runner.run_command_helper", new_callable=AsyncMock, side_effect=responses) as command, + patch("vmss_porter.runner.get_porter_outputs") as outputs, + ): + if expected_success is None: + with pytest.raises(asyncio.CancelledError): + await invoke_porter_action(message, client, config) + else: + result = await invoke_porter_action(message, client, config) + assert result is expected_success + + assert not parameters.exists() + assert not installation.exists() + assert command.await_args_list[-3].args[0] == action + assert command.await_args_list[-2].args[0] == ["porter", "parameters", "delete", "test-parameters"] + assert command.await_args_list[-1].args[0] == [ + "porter", + "installations", + "list", + "--name", + "resource", + "--output", + "json", + "--verbosity", + "error", + ] + assert command.await_args_list[-1].kwargs["log_output"] is False + outputs.assert_not_awaited() + if expected_success is None: + assert sender.send_messages.await_count == 1 + else: + assert sender.send_messages.await_count == 2 + final_status = json.loads(str(sender.send_messages.call_args.args[0])) + assert final_status["status"] == ("deleted" if expected_success else "deleting_failed") + assert "outputs" not in final_status + if expected_success: + assert "Porter confirmed that the installation is absent" in final_status["message"] + else: + assert error in final_status["message"] + + @pytest.mark.asyncio @patch("vmss_porter.runner.run_command_helper") async def test_run_porter_azure_login_failure(mock_run_command_helper): diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 38e2327066..e11154cb07 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -325,6 +325,34 @@ async def _cleanup_param_set(param_set_name: str, param_set_file: str, installat logger.debug(f"Best-effort cleanup: could not delete temp file '{path}': {e}") +async def _confirm_porter_installation_absent(installation_id: str, config: dict) -> bool: + """Confirm absence with a fresh storage query, without logging installation data.""" + try: + code, output, _ = await asyncio.wait_for( + run_command_helper( + [ + "porter", + "installations", + "list", + "--name", + installation_id, + "--output", + "json", + "--verbosity", + "error", + ], + config, + "Confirm missing installation", + log_output=False, + ), + timeout=30, + ) + return code == 0 and json.loads(output) == [] + except Exception: + # An unreadable store, invalid response or timeout does not prove absence. + return False + + async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, config: dict) -> bool: """ Handle resource message by invoking specified porter action (i.e. install, uninstall) @@ -380,7 +408,20 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf action_completed_without_error = returncode == 0 # Handle command output - if returncode != 0: + if ( + returncode != 0 + and action == "uninstall" + and "could not find installation" in (err or "") + and await _confirm_porter_installation_absent(installation_id, config) + ): + # The error text only triggers the check. A successful empty query is + # required before completing deletion without installation outputs. + action_completed_without_error = True + status_for_sb_message = statuses.pass_status_string_for[action] + resource_request_message = service_bus_message_generator( + msg_body, status_for_sb_message, "Uninstall completed: Porter confirmed that the installation is absent." + ) + elif returncode != 0: # Construct a readable representation of the command(s) command_representation = "" for cmd in porter_command: From 7fe897d307e890b1a6817178726e8f27bce0015b Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:40:43 +0100 Subject: [PATCH 04/17] Stabilise certificate provisioning and verify identity TLS (#5119) --- .../workflows/build_validation_develop.yml | 1 + CHANGELOG.md | 1 + devops/tests/test_certificate_cache.py | 311 ++++++++++++++++++ devops/tests/test_certificate_script.py | 108 ++++++ .../test_certificate_storage_readiness.py | 201 +++++++++++ .../configuring-shared-services.md | 24 ++ e2e_tests/helpers.py | 5 +- e2e_tests/unit_tests/test_identity_tls.py | 61 ++++ e2e_tests/unit_tests/test_token_refresh.py | 4 +- .../shared_services/certs/Dockerfile.tmpl | 2 +- templates/shared_services/certs/porter.yaml | 10 +- .../certs/scripts/certificate_cache.py | 232 +++++++++++++ .../certs/scripts/letsencrypt.sh | 116 ++++--- .../certs/scripts/wait_for_private_storage.py | 133 ++++++++ .../certs/terraform/staticweb.tf | 25 +- 15 files changed, 1179 insertions(+), 55 deletions(-) create mode 100644 devops/tests/test_certificate_cache.py create mode 100644 devops/tests/test_certificate_script.py create mode 100644 devops/tests/test_certificate_storage_readiness.py create mode 100644 e2e_tests/unit_tests/test_identity_tls.py create mode 100644 templates/shared_services/certs/scripts/certificate_cache.py create mode 100644 templates/shared_services/certs/scripts/wait_for_private_storage.py diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index 722c848a54..a5f94b9425 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -95,6 +95,7 @@ jobs: - '.github/workflows/pr_comment_bot.yml' - 'devops/tests/**' - 'templates/shared_services/sonatype-nexus-vm/scripts/**' + - 'templates/shared_services/certs/**' - 'templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/**' - '.github/workflows/build_validation_develop.yml' - '.github/workflows/clean_validation_envs.yml' diff --git a/CHANGELOG.md b/CHANGELOG.md index c5c33bf643..f872679036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Wait for private certificate storage readiness, reuse trusted certificates and verify TLS for E2E identity authentication. ([#5254](https://github.com/microsoft/AzureTRE/pull/5254)) * Correct managed-identity login in bundle runtimes. Preserve failed uninstall results, allow cleanup when Porter confirms that an installation is absent, and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) * Wait for management storage, retry recognised state-read permission failures and validate Terraform outputs. Protect active and recently used validation environments from scheduled cleanup. ([#5252](https://github.com/microsoft/AzureTRE/pull/5252)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/devops/tests/test_certificate_cache.py b/devops/tests/test_certificate_cache.py new file mode 100644 index 0000000000..ffcd3c8226 --- /dev/null +++ b/devops/tests/test_certificate_cache.py @@ -0,0 +1,311 @@ +"""Check certificate reuse using a local CA and mocked vault access.""" + +import base64 +from datetime import datetime, timedelta, timezone +import importlib.util +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +HELPER = Path(__file__).resolve().parents[2] / "templates/shared_services/certs/scripts/certificate_cache.py" +SPEC = importlib.util.spec_from_file_location("certificate_cache", HELPER) +CACHE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CACHE) +FQDN = "nexus-tre.example.test" + + +class CertificateCacheTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls.temporary.cleanup) + cls.root = Path(cls.temporary.name) + cls.command( + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "365", + "-subj", + "/CN=Test Root", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-keyout", + str(cls.root / "ca.key"), + "-out", + str(cls.root / "ca.pem"), + ) + for name, dns in (("valid", FQDN), ("wrong", "other.example.test"), ("wildcard", "*.example.test")): + folder = cls.root / name + folder.mkdir() + cls.command( + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=" + dns, + "-keyout", + str(folder / "privkey.pem"), + "-out", + str(folder / "request.pem"), + ) + (folder / "extensions").write_text( + "basicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\n" + "extendedKeyUsage=serverAuth\nsubjectAltName=DNS:" + dns + "\n" + ) + cls.command( + "x509", + "-req", + "-in", + str(folder / "request.pem"), + "-CA", + str(cls.root / "ca.pem"), + "-CAkey", + str(cls.root / "ca.key"), + "-CAcreateserial", + "-days", + "90", + "-extfile", + str(folder / "extensions"), + "-out", + str(folder / "fullchain.pem"), + ) + (folder / "fullchain.pem").write_text( + (folder / "fullchain.pem").read_text() + (cls.root / "ca.pem").read_text() + ) + cls.command( + "pkcs12", + "-export", + "-inkey", + str(cls.root / "valid/privkey.pem"), + "-in", + str(cls.root / "valid/fullchain.pem"), + "-out", + str(cls.root / "valid.pfx"), + "-passout", + "pass:", + ) + + @staticmethod + def command(*args): + subprocess.run(["openssl", *args], capture_output=True, check=True, timeout=20) + + def setUp(self): + temporary = self.enterContext(tempfile.TemporaryDirectory()) + self.folder = Path(temporary) + for name in ("privkey.pem", "fullchain.pem"): + shutil.copyfile(self.root / "valid" / name, self.folder / name) + self.enterContext(patch.dict(os.environ, {"SSL_CERT_FILE": str(self.root / "ca.pem")})) + self.metadata = { + "attributes": {"enabled": True}, + "policy": {"secretProperties": {"contentType": "application/x-pkcs12"}}, + "sid": "https://vault.test/secrets/retained/exact-version", + "tags": {"tre-cache": CACHE.CACHE_TAG, "fqdn": FQDN}, + "x509ThumbprintHex": "ABC", + } + + def test_valid_certificate_with_matching_key_and_public_trust_is_reusable(self): + self.assertTrue(CACHE.validate_material(self.folder, FQDN)) + + def test_near_expiry_and_expired_material_need_renewal(self): + for days in (65, 95): + with self.subTest(days=days): + self.assertFalse( + CACHE.validate_material(self.folder, FQDN, datetime.now(timezone.utc) + timedelta(days=days)) + ) + + def test_not_yet_valid_material_is_rejected(self): + with self.assertRaisesRegex(ValueError, "not yet valid"): + CACHE.validate_material(self.folder, FQDN, datetime.now(timezone.utc) - timedelta(days=1)) + + def test_wrong_hostname_and_wildcard_are_rejected(self): + for name in ("wrong", "wildcard"): + with self.subTest(name=name), self.assertRaisesRegex(ValueError, "exact requested DNS"): + CACHE.validate_material(self.root / name, FQDN) + + def test_untrusted_chain_is_rejected(self): + with patch.dict(os.environ, {"SSL_CERT_FILE": str(self.folder / "missing-ca.pem")}, clear=False): + with self.assertRaises(RuntimeError): + CACHE.validate_material(self.folder, FQDN) + + def test_mismatched_or_missing_private_key_is_rejected(self): + shutil.copyfile(self.root / "wrong/privkey.pem", self.folder / "privkey.pem") + with self.assertRaisesRegex(ValueError, "do not match"): + CACHE.validate_material(self.folder, FQDN) + (self.folder / "privkey.pem").unlink() + with self.assertRaises(RuntimeError): + CACHE.validate_material(self.folder, FQDN) + + def test_cache_identity_depends_on_hostname_not_service_alias(self): + self.assertEqual(CACHE.cache_name(FQDN), CACHE.cache_name(FQDN.upper())) + self.assertNotEqual(CACHE.cache_name(FQDN), CACHE.cache_name("other.test")) + self.assertLessEqual(len(CACHE.cache_name(FQDN)), 127) + for name in ("nexus-ssl", "nexus-e2e-12345678"): + CACHE.validate_name(name) + with self.assertRaises(ValueError): + CACHE.validate_name(CACHE.cache_name(FQDN).upper()) + + def test_absent_cache_is_the_only_lookup_error_that_allows_issuance(self): + with patch.object(CACHE, "azure", side_effect=CACHE.CertificateNotFound()) as call: + self.assertFalse(CACHE.restore("vault", FQDN, self.folder)) + self.assertEqual(call.call_count, 2) + for error in ( + RuntimeError("Forbidden"), + RuntimeError("network failure"), + FileNotFoundError("Azure CLI is missing"), + ValueError("invalid JSON"), + ): + with self.subTest(error=error), patch.object(CACHE, "azure", side_effect=error): + with self.assertRaises(type(error)): + CACHE.restore("vault", FQDN, self.folder) + + def test_deleted_disabled_or_unowned_cache_is_not_overwritten(self): + with patch.object(CACHE, "azure", side_effect=[CACHE.CertificateNotFound(), {}]): + with self.assertRaisesRegex(RuntimeError, "deleted"): + CACHE.restore("vault", FQDN, self.folder) + for replacement in ( + {"attributes": {"enabled": False}}, + {"tags": {}}, + {"tags": {"tre-cache": CACHE.CACHE_TAG, "fqdn": "other.test"}}, + ): + metadata = self.metadata | replacement + with patch.object(CACHE, "azure", return_value=metadata), self.assertRaises(ValueError): + CACHE.restore("vault", FQDN, self.folder) + + def test_restore_downloads_exact_version_and_validates_real_pfx(self): + original = CACHE.run + calls = [] + + def command(args): + calls.append(args) + if args[:4] == ["az", "keyvault", "secret", "download"]: + shutil.copyfile(self.root / "valid.pfx", self.folder / "retained.pfx") + return "" + return original(args) + + with patch.object(CACHE, "azure", return_value=self.metadata), patch.object(CACHE, "run", side_effect=command): + self.assertTrue(CACHE.restore("vault", FQDN, self.folder)) + self.assertIn(self.metadata["sid"], calls[0]) + self.assertNotIn("--name", calls[0]) + + def test_store_to_restore_round_trip_preserves_pem_chain_and_key(self): + original = CACHE.run + retained = {} + metadata = self.metadata | {"policy": {"secretProperties": {"contentType": "application/x-pem-file"}}} + restored = self.folder / "restored" + restored.mkdir() + + def vault(*args): + if args[:3] == ("keyvault", "certificate", "import"): + retained["value"] = Path(args[args.index("--file") + 1]).read_text() + else: + self.assertEqual(args[:3], ("keyvault", "certificate", "show")) + return metadata + + def command(args): + if args[:4] == ["az", "keyvault", "secret", "download"]: + self.assertIn(metadata["sid"], args) + destination = Path(args[args.index("--file") + 1]) + encoding = args[args.index("--encoding") + 1] + if encoding == "base64": + destination.write_bytes(base64.b64decode(retained["value"])) + else: + self.assertEqual(encoding, "utf-8") + destination.write_text(retained["value"]) + return "" + return original(args) + + with patch.object(CACHE, "azure", side_effect=vault), patch.object(CACHE, "run", side_effect=command): + CACHE.store("vault", FQDN, self.folder) + self.assertTrue(CACHE.restore("vault", FQDN, restored)) + self.assertEqual((restored / "fullchain.pem").read_text(), (self.folder / "fullchain.pem").read_text()) + self.assertTrue(CACHE.validate_material(restored, FQDN)) + self.assertNotIn("PRIVATE KEY", (restored / "fullchain.pem").read_text()) + + def test_unknown_or_missing_content_type_stops_before_secret_download(self): + for policy in ({}, {"secretProperties": {"contentType": "text/plain"}}): + with ( + self.subTest(policy=policy), + patch.object(CACHE, "azure", return_value=self.metadata | {"policy": policy}), + ): + with patch.object(CACHE, "run") as command, self.assertRaisesRegex(ValueError, "content type"): + CACHE.restore("vault", FQDN, self.folder) + command.assert_not_called() + + def test_incomplete_or_mismatched_retained_pem_is_rejected(self): + original = CACHE.run + metadata = self.metadata | {"policy": {"secretProperties": {"contentType": "application/x-pem-file"}}} + chain = (self.folder / "fullchain.pem").read_text() + key = (self.folder / "privkey.pem").read_text() + wrong_key = (self.root / "wrong/privkey.pem").read_text() + for name, material in (("no-key", chain), ("no-chain", key), ("mismatched-key", wrong_key + chain)): + restored = self.folder / name + restored.mkdir() + + def command(args): + if args[:4] == ["az", "keyvault", "secret", "download"]: + Path(args[args.index("--file") + 1]).write_text(material) + return "" + return original(args) + + with self.subTest(name=name), patch.object(CACHE, "azure", return_value=metadata): + with patch.object(CACHE, "run", side_effect=command), self.assertRaises((ValueError, RuntimeError)): + CACHE.restore("vault", FQDN, restored) + + def test_store_checks_material_and_retains_before_alias_publication(self): + def import_certificate(*args): + self.assertEqual(args[:3], ("keyvault", "certificate", "import")) + self.assertIn(CACHE.cache_name(FQDN), args) + self.assertIn("fqdn=" + FQDN, args) + material = Path(args[args.index("--file") + 1]).read_text() + self.assertIn("PRIVATE KEY", material) + self.assertIn("CERTIFICATE", material) + return self.metadata + + with patch.object(CACHE, "azure", side_effect=import_certificate): + CACHE.store("vault", FQDN, self.folder) + self.assertFalse((self.folder / "cache-import.pem").exists()) + + def test_failed_cache_store_removes_temporary_import_and_does_not_claim_success(self): + with patch.object(CACHE, "azure", side_effect=RuntimeError("access denied")): + with self.assertRaises(RuntimeError): + CACHE.store("vault", FQDN, self.folder) + self.assertFalse((self.folder / "cache-import.pem").exists()) + + def test_command_errors_redact_secret_output_and_preserve_not_found_distinction(self): + for detail, expected in ( + ("ERROR: (CertificateNotFound) missing", CACHE.CertificateNotFound), + ("ERROR: (Forbidden) secret-password", RuntimeError), + ("secret-private-key", RuntimeError), + ): + with patch.object( + CACHE.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["az", "secret"], stderr=detail) + ): + with self.assertRaises(expected) as caught: + CACHE.run(["az", "secret"]) + self.assertNotIn("secret-password", str(caught.exception)) + self.assertNotIn("secret-private-key", str(caught.exception)) + with patch.object(CACHE.subprocess, "run", side_effect=subprocess.TimeoutExpired("private-command", 180)): + with self.assertRaisesRegex(RuntimeError, "timed out"): + CACHE.run(["az", "secret"]) + + def test_cli_guard_rejects_reserved_alias_before_terraform(self): + result = subprocess.run( + ["python3", str(HELPER), "validate-name", "--cert-name", "tre-acme-reserved"], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/devops/tests/test_certificate_script.py b/devops/tests/test_certificate_script.py new file mode 100644 index 0000000000..6cfe1329d1 --- /dev/null +++ b/devops/tests/test_certificate_script.py @@ -0,0 +1,108 @@ +"""Check issuance, cache failure and secret handling at the shell boundary.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "templates/shared_services/certs/scripts/letsencrypt.sh" + + +class CertificateScriptTests(unittest.TestCase): + def execute(self, cache_status=0, issuance_status=0): + with tempfile.TemporaryDirectory() as temporary: + folder = Path(temporary) + log = folder / "calls.jsonl" + fake = folder / "fake" + fake.write_text("""#!/usr/bin/env python3 +import json,os,pathlib,sys +args=sys.argv[1:]; command=pathlib.Path(sys.argv[0]).name +with open(os.environ['CALL_LOG'],'a') as file: file.write(json.dumps([command,*args])+'\\n') +if command=='python3': + if 'restore' in args: sys.exit(int(os.environ['CACHE_STATUS'])) +elif command=='certbot': + if os.environ['ISSUANCE_STATUS']!='0': sys.exit(int(os.environ['ISSUANCE_STATUS'])) + d=pathlib.Path(args[args.index('--config-dir')+1])/'live'/args[args.index('--domain')+1];d.mkdir(parents=True) + for name in ('privkey.pem','fullchain.pem'): (d/name).write_text('test') +elif command=='openssl' and args[0]=='rand': print('private-password') +elif command=='az': + if args[:3]==['storage','blob','list']: print('[]') + elif args[:3]==['keyvault','certificate','import']: print(json.dumps({'sid':'https://vault.test/secrets/cert/version'})) + elif args[:3]==['keyvault','secret','set']: + assert '--output' in args and args[args.index('--output')+1]=='none' + assert pathlib.Path(args[args.index('--file')+1]).read_text()=='private-password' +""") + fake.chmod(0o700) + for name in ("python3", "certbot", "az", "openssl", "sleep"): + (folder / name).symlink_to(fake) + # The fake uses the host interpreter, not the intercepted python3 name. + fake.write_text(fake.read_text().replace("#!/usr/bin/env python3", "#!" + os.sys.executable)) + script = folder / "letsencrypt.sh" + script.write_text(SCRIPT.read_text().replace("/opt/certbot/bin/certbot", str(folder / "certbot"))) + result = subprocess.run( + [ + "bash", + str(script), + "--storage_account_name", + "storage", + "--fqdn", + "nexus.example.test", + "--keyvault_name", + "vault", + "--resource_group_name", + "rg", + "--application_gateway_name", + "agw", + "--cert_name", + "nexus-e2e-random", + "--password_name", + "password", + ], + env=os.environ + | { + "PATH": str(folder) + os.pathsep + os.environ["PATH"], + "CALL_LOG": str(log), + "CACHE_STATUS": str(cache_status), + "ISSUANCE_STATUS": str(issuance_status), + }, + capture_output=True, + text=True, + timeout=15, + ) + calls = [json.loads(line) for line in log.read_text().splitlines()] + restore = next(call for call in calls if "restore" in call) + work = Path(restore[restore.index("--directory") + 1]) + self.assertFalse(work.exists(), "Private temporary files must be removed on every exit") + self.assertNotIn("private-password", result.stdout + result.stderr) + return result, calls + + def test_reuse_does_not_call_acme_or_storage(self): + result, calls = self.execute() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call[0] == "certbot" or call[1:2] == ["storage"] for call in calls)) + self.assertTrue(any(call[1:4] == ["keyvault", "certificate", "import"] for call in calls)) + + def test_missing_cache_issues_once_and_persists_before_alias_update(self): + result, calls = self.execute(cache_status=3) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(call[0] == "certbot" for call in calls), 1) + store = next(i for i, call in enumerate(calls) if "store" in call) + alias = next(i for i, call in enumerate(calls) if call[1:4] == ["keyvault", "certificate", "import"]) + self.assertLess(store, alias) + + def test_cache_failure_stops_before_acme_and_alias_changes(self): + result, calls = self.execute(cache_status=1) + self.assertEqual(result.returncode, 1) + self.assertEqual(len(calls), 1) + + def test_issuance_failure_does_not_update_cache_or_alias(self): + result, calls = self.execute(cache_status=3, issuance_status=1) + self.assertEqual(result.returncode, 1) + self.assertFalse(any("store" in call or call[1:3] == ["keyvault", "certificate"] for call in calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/devops/tests/test_certificate_storage_readiness.py b/devops/tests/test_certificate_storage_readiness.py new file mode 100644 index 0000000000..06ff461b3b --- /dev/null +++ b/devops/tests/test_certificate_storage_readiness.py @@ -0,0 +1,201 @@ +"""Check private certificate storage readiness without contacting Azure.""" + +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import socket +import ssl +import subprocess +import sys +import unittest +from unittest.mock import MagicMock, patch + + +HELPER = Path(__file__).resolve().parents[2] / "templates/shared_services/certs/scripts/wait_for_private_storage.py" +SPEC = importlib.util.spec_from_file_location("certificate_storage_readiness", HELPER) +READINESS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(READINESS) +HOSTNAME = "certificate.blob.core.windows.net" +PRIVATE_IP = "10.0.1.27" + + +class CertificateStorageReadinessTests(unittest.TestCase): + def test_private_address_uses_verified_tls_and_storage_hostname(self): + connection = MagicMock() + context = MagicMock() + answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PRIVATE_IP, 443))] + with ( + patch.object(READINESS.socket, "getaddrinfo", return_value=answers), + patch.object(READINESS.socket, "create_connection", return_value=connection) as connect, + patch.object(READINESS.ssl, "create_default_context", return_value=context), + ): + self.assertTrue(READINESS.probe(HOSTNAME, PRIVATE_IP)[0]) + connect.assert_called_once_with((PRIVATE_IP, 443), timeout=5) + context.wrap_socket.assert_called_once_with(connection.__enter__.return_value, server_hostname=HOSTNAME) + + def test_probe_requires_tls12_and_preserves_certificate_and_hostname_checks(self): + context = ssl.create_default_context() + context.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PRIVATE_IP, 443))] + with ( + patch.object(READINESS.socket, "getaddrinfo", return_value=answers), + patch.object(READINESS.socket, "create_connection", return_value=MagicMock()), + patch.object(READINESS.ssl, "create_default_context", return_value=context), + patch.object(context, "wrap_socket", return_value=MagicMock()), + ): + self.assertTrue(READINESS.probe(HOSTNAME, PRIVATE_IP)[0]) + self.assertEqual(context.minimum_version, ssl.TLSVersion.TLSv1_2) + self.assertEqual(context.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(context.check_hostname) + + def test_public_mixed_empty_and_wrong_private_answers_never_connect(self): + for addresses in ([], ["52.239.251.164"], [PRIVATE_IP, "52.239.251.164"], ["10.0.1.28"]): + with self.subTest(addresses=addresses): + answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, 443)) for address in addresses] + with ( + patch.object(READINESS.socket, "getaddrinfo", return_value=answers), + patch.object(READINESS.socket, "create_connection") as connect, + ): + self.assertFalse(READINESS.probe(HOSTNAME, PRIVATE_IP)[0]) + connect.assert_not_called() + + def test_lookup_failure_does_not_connect(self): + with ( + patch.object(READINESS.socket, "getaddrinfo", side_effect=socket.gaierror("diagnostic detail")), + patch.object(READINESS.socket, "create_connection") as connect, + ): + ready, reason = READINESS.probe(HOSTNAME, PRIVATE_IP) + self.assertFalse(ready) + self.assertIn("gaierror", reason) + self.assertNotIn("diagnostic detail", reason) + connect.assert_not_called() + + def test_tls_verification_failure_is_not_readiness(self): + answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PRIVATE_IP, 443))] + with ( + patch.object(READINESS.socket, "getaddrinfo", return_value=answers), + patch.object( + READINESS.socket, "create_connection", side_effect=ssl.SSLCertVerificationError("private diagnostic") + ), + ): + ready, reason = READINESS.probe(HOSTNAME, PRIVATE_IP) + self.assertFalse(ready) + self.assertIn("SSLCertVerificationError", reason) + self.assertNotIn("private diagnostic", reason) + + def wait_with_results(self, results, **kwargs): + elapsed = [0.0] + + def sleep(seconds): + elapsed[0] += seconds + + with ( + patch.object(READINESS.time, "monotonic", side_effect=lambda: elapsed[0]), + patch.object(READINESS.time, "sleep", side_effect=sleep), + patch.object(READINESS, "run_probe", side_effect=results) as probe, + contextlib.redirect_stdout(io.StringIO()), + ): + READINESS.wait_for_private_storage(HOSTNAME, PRIVATE_IP, **kwargs) + return probe.call_count, elapsed[0] + + def test_observed_dns_changes_restart_all_consecutive_checks(self): + # Private, lookup error, private, public, private, public, then stable. + results = [ + (ready, "private" if ready else "lookup or public DNS") + for ready in [True, False, True, False, True, False] + [True] * 7 + ] + count, elapsed = self.wait_with_results(results) + self.assertEqual(count, 13) + self.assertEqual(elapsed, 60) + + def test_sustained_readiness_spans_at_least_thirty_seconds(self): + count, elapsed = self.wait_with_results([(True, "ready")] * 7) + self.assertEqual(count, 7) + self.assertEqual(elapsed, 30) + + def test_tls_failure_also_restarts_stability(self): + results = [(True, "ready")] * 6 + [(False, "TLS check failed")] + [(True, "ready")] * 7 + count, elapsed = self.wait_with_results(results) + self.assertEqual(count, 14) + self.assertEqual(elapsed, 65) + + def test_timeout_prevents_unbounded_wait(self): + with self.assertRaisesRegex(TimeoutError, "within 12 seconds"): + self.wait_with_results([(False, "not ready")] * 3, timeout=12) + + def test_probe_gets_only_remaining_deadline(self): + elapsed = [0.0] + + def consume_budget(hostname, expected_ip, timeout): + self.assertEqual(timeout, 2) + elapsed[0] += timeout + return True, "too late" + + with ( + patch.object(READINESS.time, "monotonic", side_effect=lambda: elapsed[0]), + patch.object(READINESS, "run_probe", side_effect=consume_budget), + patch.object(READINESS.time, "sleep") as sleep, + ): + with self.assertRaises(TimeoutError): + READINESS.wait_for_private_storage(HOSTNAME, PRIVATE_IP, timeout=2, required_successes=1) + sleep.assert_not_called() + + def test_blocked_resolver_child_has_finite_timeout(self): + with patch.object(READINESS.subprocess, "run", side_effect=subprocess.TimeoutExpired("probe", 2)) as run: + ready, reason = READINESS.run_probe(HOSTNAME, PRIVATE_IP, 2) + self.assertFalse(ready) + self.assertIn("timed out", reason) + self.assertEqual(run.call_args.kwargs["timeout"], 2) + self.assertFalse(run.call_args.kwargs.get("shell", False)) + + def test_invalid_child_results_cannot_report_readiness(self): + for value, code in [ + ("not json", 0), + ("[]", 0), + ('{"ready":"yes","reason":"x"}', 0), + ('{"ready":true,"reason":"x"}', 1), + ("", 2), + ]: + with self.subTest(value=value, code=code): + result = subprocess.CompletedProcess([], code, value, "internal diagnostic") + with patch.object(READINESS.subprocess, "run", return_value=result): + ready, reason = READINESS.run_probe(HOSTNAME, PRIVATE_IP, 2) + self.assertFalse(ready) + self.assertNotIn("internal diagnostic", reason) + + def test_cli_rejects_public_and_special_expected_addresses(self): + for address in ("52.239.251.164", "127.0.0.1", "169.254.169.254", "0.0.0.0", "invalid"): + with self.subTest(address=address): + result = subprocess.run( + [sys.executable, str(HELPER), "--hostname", HOSTNAME, "--expected-ip", address], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("private endpoint IP address", result.stderr) + + def test_cli_failure_returns_nonzero(self): + with ( + patch.object(sys, "argv", [str(HELPER), "--hostname", HOSTNAME, "--expected-ip", PRIVATE_IP]), + patch.object(READINESS, "wait_for_private_storage", side_effect=TimeoutError("not ready")), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(READINESS.main(), 1) + + def test_probe_mode_reports_json_for_parent(self): + output = io.StringIO() + with ( + patch.object(sys, "argv", [str(HELPER), "--probe", "--hostname", HOSTNAME, "--expected-ip", PRIVATE_IP]), + patch.object(READINESS, "probe", return_value=(False, "not ready")), + contextlib.redirect_stdout(output), + ): + self.assertEqual(READINESS.main(), 1) + self.assertEqual(json.loads(output.getvalue()), {"ready": False, "reason": "not ready"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index 2fb39f441b..b81880de35 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -30,6 +30,30 @@ Once deployed, the certs service will use Letsencrypt to generate a certificate You can verify whether this has been successful by navigating to your core keyvault (`kv-{TRE_ID}`) and looking for a certificate called `nexus-ssl` (or whatever you called it). +### Certificate reuse and renewal + +From certificate bundle `0.7.16`, install and `renew` reuse a retained certificate when more than 30 days remain before expiry. +Version `0.7.16` can also reuse certificates retained by version `0.7.15`. +The bundle verifies public trust, the exact DNS name and the private key before reuse. +It issues a new certificate when no retained certificate exists or renewal is due. +A vault lookup, ownership or certificate validation failure stops the operation. It does not trigger another issuance attempt. + +The core Key Vault retains one certificate named `tre-acme-` for each hostname. +The `tre-cache=certificate-v1` and `fqdn` tags identify this certificate. +The certificate includes an exportable private key and uses the existing vault access controls. +Names starting with `tre-acme-` are reserved. Use another name for the service certificate. + +Uninstall removes the service certificate and its infrastructure. It retains the hostname certificate for later installs, including installs with a different service certificate name. +Deleting the core vault removes this retained material under the vault's existing soft-delete and purge-protection rules. +If an administrator disables or deletes the retained certificate, investigate that change before enabling or recovering it. +Do not remove the cache to bypass an issuance limit. Compromised certificates require revocation and replacement through the normal incident procedure. + +The cache starts with the first successful issuance by a bundle that supports certificate retention. It does not search for or adopt uncached certificates. +An ACME rate-limit response still fails the operation. Follow the retry time in that response. +Run only one certificate lifecycle or dependent test suite per environment at a time. + +### Deploy Nexus + After verifying the certificate has been generated, you can deploy Nexus: 1. Run the below command in your terminal to build, publish and register the Nexus shared service bundle: diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index 6f111b9c89..52d5c861d0 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -114,13 +114,14 @@ async def get_admin_token(verify) -> TokenProvider: def get_token(scope_uri, verify) -> TokenProvider: + """Keep the TRE API's ``verify`` argument for callers, but always verify identity TLS.""" if config.TEST_ACCOUNT_CLIENT_ID != "" and config.TEST_ACCOUNT_CLIENT_SECRET != "": # Logging in as an Enterprise Application: Use Client Credentials flow credential = ClientSecretCredential( config.AAD_TENANT_ID, config.TEST_ACCOUNT_CLIENT_ID, config.TEST_ACCOUNT_CLIENT_SECRET, - connection_verify=verify, + connection_verify=True, authority=cloud.get_aad_authority_fqdn(), ) scope = f"{scope_uri}/.default" @@ -130,7 +131,7 @@ def get_token(scope_uri, verify) -> TokenProvider: config.TEST_APP_ID, config.TEST_USER_NAME, config.TEST_USER_PASSWORD, - connection_verify=verify, + connection_verify=True, authority=cloud.get_aad_authority_fqdn(), tenant_id=config.AAD_TENANT_ID, ) diff --git a/e2e_tests/unit_tests/test_identity_tls.py b/e2e_tests/unit_tests/test_identity_tls.py new file mode 100644 index 0000000000..e0e0edfa84 --- /dev/null +++ b/e2e_tests/unit_tests/test_identity_tls.py @@ -0,0 +1,61 @@ +"""Check identity TLS at the real SDK transport without making network requests.""" + +import os +import unittest +from unittest.mock import patch + +from azure.core.exceptions import ClientAuthenticationError + +from e2e_tests import helpers + + +class OfflineTransportReached(Exception): + pass + + +class IdentityTlsTests(unittest.TestCase): + def setUp(self): + self.enterContext( + patch.multiple( + helpers.config, + AAD_TENANT_ID="tenant", + TEST_ACCOUNT_CLIENT_ID="client", + TEST_ACCOUNT_CLIENT_SECRET="dummy-secret", + TEST_APP_ID="app", + TEST_USER_NAME="user", + TEST_USER_PASSWORD="dummy-password", + ) + ) + self.enterContext( + patch.object(helpers.cloud, "get_aad_authority_fqdn", return_value="login.microsoftonline.com") + ) + + def check_identity_tls(self): + for tre_verify in (True, False): + for ca_bundle in ("", "/configured/identity-ca.pem"): + with ( + self.subTest(tre_verify=tre_verify, ca_bundle=ca_bundle), + patch.dict(os.environ, {"REQUESTS_CA_BUNDLE": ca_bundle, "CURL_CA_BUNDLE": ""}), + patch( + "requests.adapters.HTTPAdapter.send", + side_effect=OfflineTransportReached("offline identity TLS test"), + ) as send, + ): + provider = helpers.get_token("api://workspace", verify=tre_verify) + try: + # Let Azure Identity and Requests resolve the transport settings. + # Stop before the adapter opens a connection or validates the CA file. + with self.assertRaisesRegex(ClientAuthenticationError, "offline identity TLS test"): + provider.get_token() + send.assert_called_once() + self.assertTrue(send.call_args.args[0].url.startswith("https://login.microsoftonline.com/")) + self.assertEqual(send.call_args.kwargs["verify"], ca_bundle or True) + finally: + provider._credential.close() + + def test_client_credentials_verify_identity_tls_independently_of_tre(self): + self.check_identity_tls() + + def test_password_credentials_verify_identity_tls_independently_of_tre(self): + with patch.multiple(helpers.config, TEST_ACCOUNT_CLIENT_ID="", TEST_ACCOUNT_CLIENT_SECRET=""): + self.check_identity_tls() diff --git a/e2e_tests/unit_tests/test_token_refresh.py b/e2e_tests/unit_tests/test_token_refresh.py index 93917231d9..17e6194222 100644 --- a/e2e_tests/unit_tests/test_token_refresh.py +++ b/e2e_tests/unit_tests/test_token_refresh.py @@ -82,7 +82,7 @@ def test_client_credentials_reuse_credential_and_scope(self): [call.args for call in credential.get_token.call_args_list], [("api://workspace/.default",)] * 2 ) - def test_password_credentials_keep_user_scope_and_tls_setting(self): + def test_password_credentials_keep_user_scope_and_verify_identity_tls(self): with ( patch.multiple( helpers.config, @@ -103,7 +103,7 @@ def test_password_credentials_keep_user_scope_and_tls_setting(self): "app", "user", "dummy-password", - connection_verify=False, + connection_verify=True, authority="login.example.test", tenant_id="tenant", ) diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index f61b684795..3084917f6b 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -10,7 +10,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa # Install jq + git (git is required for terraform_azurerm_environment_configuration) RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ apt-get update \ - && apt-get install -y --no-install-recommends git jq + && apt-get install -y --no-install-recommends git jq openssl ca-certificates # Install Certbot RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 2ce29f568c..0cbb66a7a9 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-certs -version: 0.7.13 +version: 0.7.16 description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" registry: azuretre dockerfile: Dockerfile.tmpl @@ -67,6 +67,14 @@ mixins: clientVersion: 2.81.0 install: + - exec: + description: "Validate the certificate name" + command: python3 + arguments: + - ./scripts/certificate_cache.py + - validate-name + flags: + cert-name: ${ bundle.parameters.cert_name } - terraform: description: "Deploy shared service" vars: diff --git a/templates/shared_services/certs/scripts/certificate_cache.py b/templates/shared_services/certs/scripts/certificate_cache.py new file mode 100644 index 0000000000..5a996a9dcf --- /dev/null +++ b/templates/shared_services/certs/scripts/certificate_cache.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Retain verified certificates in the core vault across service replacement.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +from datetime import datetime, timezone + +CACHE_PREFIX = "tre-acme-" +CACHE_TAG = "certificate-v1" +RENEW_BEFORE_SECONDS = 30 * 24 * 60 * 60 +CACHE_MISS = 3 + + +class CertificateNotFound(RuntimeError): + """The vault returned its explicit certificate-not-found response.""" + + +def cache_name(fqdn): + return CACHE_PREFIX + hashlib.sha256(fqdn.lower().encode("ascii")).hexdigest() + + +def run(arguments): + """Do not expose command output or arguments when a secret operation fails.""" + try: + return subprocess.run(arguments, capture_output=True, text=True, timeout=180, check=True).stdout + except subprocess.CalledProcessError as error: + code = re.search(r"^\s*(?:ERROR: )?\(([A-Za-z0-9]+)\)", error.stderr or "", re.MULTILINE) + if code and code[1] == "CertificateNotFound": + raise CertificateNotFound("Certificate not found") from None + raise RuntimeError(f"Certificate command failed ({code[1] if code else 'see operation diagnostics'})") from None + except subprocess.TimeoutExpired: + raise RuntimeError("Certificate command timed out") from None + + +def azure(*arguments): + return json.loads(run(["az", *arguments, "--only-show-errors", "--output", "json"])) + + +def validate_name(name): + if name.lower().startswith(CACHE_PREFIX): + raise ValueError(f"Certificate names starting with {CACHE_PREFIX} are reserved for the retained cache") + + +def validate_material(directory, fqdn, now=None): + """Check exact SAN, public trust, validity and the private key before reuse.""" + leaf = directory / "leaf.pem" + fullchain = directory / "fullchain.pem" + key = directory / "privkey.pem" + run(["openssl", "x509", "-in", str(fullchain), "-out", str(leaf)]) + san = run(["openssl", "x509", "-in", str(leaf), "-noout", "-ext", "subjectAltName"]) + names = re.findall(r"DNS:([^,\s]+)", san) + if fqdn.lower() not in [name.lower() for name in names]: + raise ValueError("Certificate does not contain the exact requested DNS name") + dates = run(["openssl", "x509", "-in", str(leaf), "-noout", "-dates"]) + bounds = dict(line.split("=", 1) for line in dates.strip().splitlines()) + start, end = [ + datetime.strptime(bounds[k], "%b %d %H:%M:%S %Y %Z").replace(tzinfo=timezone.utc) + for k in ("notBefore", "notAfter") + ] + now = now or datetime.now(timezone.utc) + if start > now: + raise ValueError("Certificate is not yet valid") + certificate_key = run(["openssl", "x509", "-in", str(leaf), "-noout", "-pubkey"]) + private_key = run(["openssl", "pkey", "-in", str(key), "-pubout"]) + if certificate_key.strip() != private_key.strip(): + raise ValueError("Certificate and private key do not match") + # Verify the chain at an instant within the certificate validity period. + # Expired but otherwise valid material requests renewal below. + verify_time = now.timestamp() if now < end else end.timestamp() - 1 + run( + [ + "openssl", + "verify", + "-purpose", + "sslserver", + "-verify_hostname", + fqdn, + "-attime", + str(int(verify_time)), + "-untrusted", + str(fullchain), + str(leaf), + ] + ) + return (end - now).total_seconds() > RENEW_BEFORE_SECONDS + + +def restore(vault, fqdn, directory): + name = cache_name(fqdn) + try: + certificate = azure("keyvault", "certificate", "show", "--vault-name", vault, "--name", name) + except CertificateNotFound: + # A deleted cache is an explicit administrative state, not permission to issue again. + try: + azure("keyvault", "certificate", "show-deleted", "--vault-name", vault, "--name", name) + except CertificateNotFound: + print("No retained certificate. Production issuance is required.") + return False + raise RuntimeError("The retained certificate is deleted. Recover it before retrying") + if certificate.get("attributes", {}).get("enabled") is not True: + raise ValueError("The retained certificate is disabled") + if certificate.get("tags", {}).get("tre-cache") != CACHE_TAG: + raise ValueError("The retained certificate has no recognised ownership tag") + if certificate.get("tags", {}).get("fqdn") != fqdn.lower(): + raise ValueError("The retained certificate belongs to another DNS name") + content_type = certificate.get("policy", {}).get("secretProperties", {}).get("contentType") + if content_type not in ("application/x-pem-file", "application/x-pkcs12"): + raise ValueError("The retained certificate has an unsupported or missing content type") + is_pem = content_type == "application/x-pem-file" + # Download the exact version in its stored format. PEM secrets contain text; + # PFX secrets contain base64 data. Downloading does not convert the format. + material = directory / ("retained.pem" if is_pem else "retained.pfx") + run( + [ + "az", + "keyvault", + "secret", + "download", + "--id", + certificate["sid"], + "--file", + str(material), + "--encoding", + "utf-8" if is_pem else "base64", + "--only-show-errors", + ] + ) + if is_pem: + certificates = re.findall( + r"-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", material.read_text(), re.DOTALL + ) + if not certificates: + raise ValueError("The retained PEM contains no certificates") + (directory / "fullchain.pem").write_text("\n".join(certificates) + "\n") + run(["openssl", "pkey", "-in", str(material), "-out", str(directory / "privkey.pem")]) + else: + run( + [ + "openssl", + "pkcs12", + "-in", + str(material), + "-passin", + "pass:", + "-nokeys", + "-out", + str(directory / "fullchain.pem"), + ] + ) + run( + [ + "openssl", + "pkcs12", + "-in", + str(material), + "-passin", + "pass:", + "-nocerts", + "-nodes", + "-out", + str(directory / "privkey.pem"), + ] + ) + if not validate_material(directory, fqdn): + print("The retained certificate is due for renewal. Production issuance is required.") + return False + print(f"Reusing retained certificate {name}, thumbprint {certificate['x509ThumbprintHex']}") + return True + + +def store(vault, fqdn, directory): + if not validate_material(directory, fqdn): + raise ValueError("The issued certificate is already due for renewal") + # A PEM import includes the private key and complete chain. The CLI reads a private file. + material = directory / "cache-import.pem" + material.write_text((directory / "privkey.pem").read_text() + (directory / "fullchain.pem").read_text()) + try: + certificate = azure( + "keyvault", + "certificate", + "import", + "--vault-name", + vault, + "--name", + cache_name(fqdn), + "--file", + str(material), + "--tags", + f"tre-cache={CACHE_TAG}", + f"fqdn={fqdn.lower()}", + ) + finally: + material.unlink(missing_ok=True) + print(f"Retained issued certificate {cache_name(fqdn)}, thumbprint {certificate['x509ThumbprintHex']}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("validate-name", "restore", "store")) + parser.add_argument("--cert-name", required=True) + parser.add_argument("--vault") + parser.add_argument("--fqdn") + parser.add_argument("--directory", type=Path) + args = parser.parse_args() + os.umask(0o077) + try: + validate_name(args.cert_name) + if args.action == "validate-name": + return 0 + if not args.vault or not args.fqdn or not args.directory: + parser.error("--vault, --fqdn and --directory are required") + if args.action == "restore": + return 0 if restore(args.vault, args.fqdn, args.directory) else CACHE_MISS + store(args.vault, args.fqdn, args.directory) + return 0 + except (ValueError, KeyError, OSError, RuntimeError): + # Exception arguments can contain secret command results or certificate material. + print( + "Certificate reuse failed. Check vault access, cache ownership, validity and trust before retrying.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/shared_services/certs/scripts/letsencrypt.sh b/templates/shared_services/certs/scripts/letsencrypt.sh index bd88397215..110a56301c 100755 --- a/templates/shared_services/certs/scripts/letsencrypt.sh +++ b/templates/shared_services/certs/scripts/letsencrypt.sh @@ -1,5 +1,7 @@ #!/bin/bash set -e +set -o pipefail +umask 077 script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") @@ -50,71 +52,88 @@ done # done with processing args and can set this set -o nounset -echo "Checking for index.html file in storage account" +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT +cache_args=(--cert-name "$cert_name" --vault "$keyvault_name" --fqdn "$fqdn" --directory "$work_dir") -# Create the default index.html page -cat << EOF > index.html +if python3 "${script_dir}/certificate_cache.py" restore "${cache_args[@]}"; then + echo "Using the retained certificate without an ACME request" +else + cache_status=$? + if [[ "$cache_status" -ne 3 ]]; then + exit "$cache_status" + fi + echo "Checking for index.html file in storage account" + + # Create the default index.html page + cat << EOF > "${work_dir}/index.html" EOF -# shellcheck disable=SC2016 -indexExists=$(az storage blob list -o json \ - --account-name "${storage_account_name}" \ - --auth-mode login \ - --container-name '$web' \ - --query "[?name=='index.html'].name" \ - | jq 'length') - -if [[ ${indexExists} -lt 1 ]]; then - echo "No existing file found. Uploading index.html file" - # shellcheck disable=SC2016 - az storage blob upload \ + indexExists=$(az storage blob list -o json \ --account-name "${storage_account_name}" \ --auth-mode login \ --container-name '$web' \ - --file index.html \ - --name index.html \ - --no-progress \ - --only-show-errors - - # Wait a bit for the App Gateway health probe to notice - echo "Waiting 30s for app gateway health probe" - sleep 30s -else - echo "index.html already present" -fi - -ledir="${script_dir}/../letsencrypt" -mkdir -p "${ledir}/logs" - -# Initiate the ACME challange -echo "Initiating ACME challenge" -export STORAGE_ACCOUNT_NAME="${storage_account_name}" -/opt/certbot/bin/certbot certonly \ - --config-dir "${ledir}" \ - --work-dir "${ledir}" \ - --logs-dir "${ledir}"/logs \ - --manual \ - --preferred-challenges=http \ - --manual-auth-hook "${script_dir}"/auth-hook.sh \ - --manual-cleanup-hook "${script_dir}"/cleanup-hook.sh \ - --domain "${fqdn}" \ - --non-interactive \ - --agree-tos \ - --register-unsafely-without-email + --query "[?name=='index.html'].name" \ + | jq 'length') + + if [[ ${indexExists} -lt 1 ]]; then + echo "No existing file found. Uploading index.html file" + + # shellcheck disable=SC2016 + az storage blob upload \ + --account-name "${storage_account_name}" \ + --auth-mode login \ + --container-name '$web' \ + --file "${work_dir}/index.html" \ + --name index.html \ + --no-progress \ + --only-show-errors + + # Wait a bit for the App Gateway health probe to notice + echo "Waiting 30s for app gateway health probe" + sleep 30s + else + echo "index.html already present" + fi + ledir="${work_dir}/letsencrypt" + mkdir -p "${ledir}/logs" + + # Initiate the ACME challenge + echo "Initiating ACME challenge" + export STORAGE_ACCOUNT_NAME="${storage_account_name}" + /opt/certbot/bin/certbot certonly \ + --config-dir "${ledir}" \ + --work-dir "${ledir}" \ + --logs-dir "${ledir}"/logs \ + --manual \ + --preferred-challenges=http \ + --manual-auth-hook "${script_dir}"/auth-hook.sh \ + --manual-cleanup-hook "${script_dir}"/cleanup-hook.sh \ + --domain "${fqdn}" \ + --non-interactive \ + --agree-tos \ + --register-unsafely-without-email + + cp "${ledir}/live/${fqdn}/privkey.pem" "${work_dir}/privkey.pem" + cp "${ledir}/live/${fqdn}/fullchain.pem" "${work_dir}/fullchain.pem" + # Persist issuance before updating the service alias. A later failure can reuse it. + python3 "${script_dir}/certificate_cache.py" store "${cache_args[@]}" +fi # Convert the generated certificate to a .pfx echo "Got cert. Converting to PFX" -CERT_DIR="${ledir}/live/${fqdn}" +CERT_DIR="${work_dir}" CERT_PASSWORD=$(openssl rand -base64 30) +printf %s "$CERT_PASSWORD" > "${CERT_DIR}/password" openssl pkcs12 -export \ -inkey "${CERT_DIR}/privkey.pem" \ -in "${CERT_DIR}/fullchain.pem" \ -out "${CERT_DIR}/aci.pfx" \ - -passout "pass:${CERT_PASSWORD}" + -passout "file:${CERT_DIR}/password" # Save cert and password to KeyVault echo "Importing cert to KeyVault ${keyvault_name}" @@ -129,7 +148,8 @@ sid=$(az keyvault certificate import \ echo "Saving certificate password to KV with key ${password_name}" az keyvault secret set --name "$password_name" \ --vault-name "${keyvault_name}" \ - --value "${CERT_PASSWORD}" + --file "${CERT_DIR}/password" \ + --output none --only-show-errors echo "Updating SSL cert in app gateway" az network application-gateway ssl-cert update \ diff --git a/templates/shared_services/certs/scripts/wait_for_private_storage.py b/templates/shared_services/certs/scripts/wait_for_private_storage.py new file mode 100644 index 0000000000..c357930d99 --- /dev/null +++ b/templates/shared_services/certs/scripts/wait_for_private_storage.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Wait for consecutive private DNS and verified TLS checks before storage writes.""" + +import argparse +import ipaddress +import json +import os +from pathlib import Path +import socket +import ssl +import subprocess +import sys +import time + + +PROBE_TIMEOUT_SECONDS = 10 +TIMEOUT_SECONDS = 300 +REQUIRED_SUCCESSES = 7 +INTERVAL_SECONDS = 5 + + +def probe(hostname, expected_ip): + """Connect only when every DNS answer matches the private endpoint address.""" + try: + answers = { + ipaddress.ip_address(item[4][0]) for item in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM) + } + if answers != {ipaddress.ip_address(expected_ip)}: + return False, "DNS does not exclusively resolve to the expected private endpoint" + context = ssl.create_default_context() + context.minimum_version = ssl.TLSVersion.TLSv1_2 + with socket.create_connection((expected_ip, 443), timeout=5) as connection: + with context.wrap_socket(connection, server_hostname=hostname): + return True, "private DNS and TLS verification passed" + except (OSError, ValueError) as error: + return False, f"DNS or TLS check failed ({type(error).__name__})" + + +def run_probe(hostname, expected_ip, timeout): + # A child process bounds getaddrinfo(), which has no per-call timeout. + try: + result = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + "--probe", + "--hostname", + hostname, + "--expected-ip", + expected_ip, + ], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + return False, "DNS or TLS check timed out" + if result.returncode not in (0, 1): + return False, "DNS or TLS check process failed" + try: + value = json.loads(result.stdout) + if type(value.get("ready")) is not bool or not isinstance(value.get("reason"), str): + raise ValueError("Invalid probe result") + except (AttributeError, ValueError): + return False, "DNS or TLS check returned an invalid result" + if value["ready"] != (result.returncode == 0): + return False, "DNS or TLS check returned an inconsistent result" + return value["ready"], value["reason"] + + +def wait_for_private_storage( + hostname, expected_ip, timeout=TIMEOUT_SECONDS, required_successes=REQUIRED_SUCCESSES, interval=INTERVAL_SECONDS +): + deadline = time.monotonic() + timeout + successes = 0 + last_reason = "no completed check" + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + ready, last_reason = run_probe(hostname, expected_ip, min(PROBE_TIMEOUT_SECONDS, remaining)) + if time.monotonic() >= deadline: + break + successes = successes + 1 if ready else 0 + print(f"Storage readiness: {last_reason} ({successes}/{required_successes} consecutive checks)", flush=True) + if successes >= required_successes: + return + time.sleep(min(interval, max(0, deadline - time.monotonic()))) + raise TimeoutError( + f"Private storage endpoint was not ready within {timeout} seconds: {last_reason}. Check private DNS and endpoint connectivity before retrying." + ) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--hostname", default=os.environ.get("STORAGE_BLOB_HOST")) + parser.add_argument("--expected-ip", default=os.environ.get("STORAGE_PRIVATE_IP")) + parser.add_argument("--probe", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + if not args.hostname or any(c.isspace() for c in args.hostname) or any(c in args.hostname for c in "/:\\"): + parser.error("A storage DNS hostname is required") + try: + address = ipaddress.ip_address(args.expected_ip) + except ValueError: + parser.error("A private endpoint IP address is required") + if ( + not address.is_private + or address.is_loopback + or address.is_link_local + or address.is_unspecified + or address.is_multicast + ): + parser.error("The expected address must be a private endpoint IP address") + return args + + +def main(): + args = parse_args() + if args.probe: + ready, reason = probe(args.hostname, args.expected_ip) + print(json.dumps({"ready": ready, "reason": reason})) + return 0 if ready else 1 + try: + wait_for_private_storage(args.hostname, args.expected_ip) + except (OSError, TimeoutError) as error: + print(f"Error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/shared_services/certs/terraform/staticweb.tf b/templates/shared_services/certs/terraform/staticweb.tf index ee8093ed53..1d5b97c3a0 100644 --- a/templates/shared_services/certs/terraform/staticweb.tf +++ b/templates/shared_services/certs/terraform/staticweb.tf @@ -47,7 +47,30 @@ resource "azurerm_storage_account_static_website" "staticweb_site" { index_document = "index.html" error_404_document = "404.html" - depends_on = [azurerm_private_endpoint.blobpe, azurerm_private_endpoint.webpe] + depends_on = [terraform_data.staticweb_private_endpoint_ready] +} + +# Endpoint creation can finish before the DNS answers become stable. +resource "terraform_data" "staticweb_private_endpoint_ready" { + triggers_replace = { + endpoint = azurerm_private_endpoint.blobpe.id + address = azurerm_private_endpoint.blobpe.private_service_connection[0].private_ip_address + helper = filesha256("${path.module}/../scripts/wait_for_private_storage.py") + } + + lifecycle { + replace_triggered_by = [azurerm_private_endpoint.blobpe] + } + + depends_on = [azurerm_private_endpoint.webpe] + + provisioner "local-exec" { + command = "python3 \"${path.module}/../scripts/wait_for_private_storage.py\"" + environment = { + STORAGE_BLOB_HOST = azurerm_storage_account.staticweb.primary_blob_host + STORAGE_PRIVATE_IP = azurerm_private_endpoint.blobpe.private_service_connection[0].private_ip_address + } + } } resource "azurerm_role_assignment" "stgwriter" { From a7bb4b8295a7947a6f01c95c0fdb45a22874d851 Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:41:24 +0100 Subject: [PATCH 05/17] Fix Airlock VM prerequisites and shared-service test cleanup (#5119) --- .github/scripts/build.js | 31 +- .github/scripts/build.test.js | 172 ++++++- .../scripts/ci_environment_workflow.test.js | 88 +++- .github/scripts/package-lock.json | 4 +- .github/scripts/package.json | 2 +- .github/workflows/deploy_tre.yml | 8 +- .github/workflows/deploy_tre_branch.yml | 6 + .github/workflows/deploy_tre_reusable.yml | 59 ++- .github/workflows/pr_comment_bot.yml | 2 + CHANGELOG.md | 1 + Makefile | 7 + devops/tests/test_review_vm_prerequisites.py | 123 +++++ devops/version.txt | 2 +- docs/tre-developers/github-pr-bot-commands.md | 51 ++- e2e_tests/airlock/request.py | 37 ++ e2e_tests/ci_groups.py | 139 ++++++ e2e_tests/config.py | 3 + e2e_tests/conftest.py | 176 +++++-- e2e_tests/pytest.ini | 3 + e2e_tests/resources/nexus.py | 111 +++++ e2e_tests/resources/resource.py | 158 +++++-- e2e_tests/test_airlock.py | 154 +++++-- e2e_tests/test_guacamole_linuxvm.py | 78 +--- e2e_tests/test_shared_services.py | 212 +++++++-- e2e_tests/timeouts.py | 40 ++ e2e_tests/unit_tests/requirements.txt | 2 + .../test_accepted_resource_cleanup.py | 206 +++++++++ e2e_tests/unit_tests/test_airlock_request.py | 111 +++++ e2e_tests/unit_tests/test_ci_groups.py | 134 ++++++ e2e_tests/unit_tests/test_disable_deadline.py | 60 +++ e2e_tests/unit_tests/test_firewall_patch.py | 86 ++++ e2e_tests/unit_tests/test_fixture_cleanup.py | 432 ++++++++++++++++++ .../test_guacamole_linuxvm_cleanup.py | 44 +- e2e_tests/unit_tests/test_linux_vm_nexus.py | 92 +++- .../unit_tests/test_main_test_selection.py | 54 +++ .../unit_tests/test_review_vm_cleanup.py | 252 ++++++++++ .../test_review_workspace_ownership.py | 76 +++ .../test_shared_service_admin_vm.py | 32 ++ .../unit_tests/test_shared_service_nexus.py | 297 ++++++++++++ .../porter.yaml | 10 +- .../porter.yaml | 10 +- .../terraform/windowsvm.tf | 1 + .../guacamole-azure-linuxvm/porter.yaml | 10 +- .../guacamole-azure-windowsvm/porter.yaml | 2 +- .../terraform/vm/locals.tf | 1 + .../terraform/vm/variables.tf | 5 + .../terraform/vm/vm_config.ps1 | 40 +- 47 files changed, 3321 insertions(+), 303 deletions(-) create mode 100644 devops/tests/test_review_vm_prerequisites.py create mode 100644 e2e_tests/ci_groups.py create mode 100644 e2e_tests/resources/nexus.py create mode 100644 e2e_tests/timeouts.py create mode 100644 e2e_tests/unit_tests/test_accepted_resource_cleanup.py create mode 100644 e2e_tests/unit_tests/test_airlock_request.py create mode 100644 e2e_tests/unit_tests/test_ci_groups.py create mode 100644 e2e_tests/unit_tests/test_disable_deadline.py create mode 100644 e2e_tests/unit_tests/test_firewall_patch.py create mode 100644 e2e_tests/unit_tests/test_fixture_cleanup.py create mode 100644 e2e_tests/unit_tests/test_main_test_selection.py create mode 100644 e2e_tests/unit_tests/test_review_vm_cleanup.py create mode 100644 e2e_tests/unit_tests/test_review_workspace_ownership.py create mode 100644 e2e_tests/unit_tests/test_shared_service_admin_vm.py create mode 100644 e2e_tests/unit_tests/test_shared_service_nexus.py diff --git a/.github/scripts/build.js b/.github/scripts/build.js index 74ae7bc491..8c00b25df9 100644 --- a/.github/scripts/build.js +++ b/.github/scripts/build.js @@ -7,6 +7,7 @@ const { createHash } = require('crypto'); const SKIP_DEPLOYMENT_FLAG = "skip_deployment"; +const ACCEPT_NEXUS_EULA_FLAG = "accept_nexus_eula"; async function getCommandFromComment({ core, context, github }) { const commentUsername = context.payload.comment.user.login; @@ -54,6 +55,7 @@ async function getCommandFromComment({ core, context, github }) { const commentFirstLine = commentBody.split("\n")[0]; let command = "none"; let skipDeployment = false; + let acceptNexusEula = false; const trimmedFirstLine = commentFirstLine.trim(); if (trimmedFirstLine[0] === "/") { // only allow actions for users with write access @@ -122,9 +124,17 @@ async function getCommandFromComment({ core, context, github }) { break; } + const consentFlags = parts.slice(1).filter(part => part.startsWith(ACCEPT_NEXUS_EULA_FLAG)); + if (consentFlags.length !== 1 || consentFlags[0] !== ACCEPT_NEXUS_EULA_FLAG) { + const message = `:warning: Nexus CE EULA acceptance is required for shared service tests. No deployment or tests were started. After accepting the EULA, re-run \`${commandText} [] ${ACCEPT_NEXUS_EULA_FLAG}\`.`; + await addActionComment({ github }, repoOwner, repoName, prNumber, commentUsername, commentLink, message); + break; + } + const runTests = await handleTestCommand({ core, github }, parts, "shared service tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink }); if (runTests) { command = "run-tests-shared-services"; + acceptNexusEula = true; } break; } @@ -145,9 +155,23 @@ async function getCommandFromComment({ core, context, github }) { case "/test-airlock": { + if (commandHasSkipDeploymentFlag(parts)) { + await addUnsupportedSkipDeploymentFlagComment({ github }, repoOwner, repoName, prNumber, commentUsername, commentLink, commandText); + break; + } + + const consentFlags = parts.slice(1).filter(part => part === ACCEPT_NEXUS_EULA_FLAG); + const shaParts = parts.slice(1).filter(part => part !== ACCEPT_NEXUS_EULA_FLAG); + if (consentFlags.length > 1 || shaParts.length > 1 || shaParts.some(part => !/^[0-9a-f]{7,40}$/i.test(part))) { + const message = `:warning: Invalid Airlock arguments. No deployment or tests were started. Use \`${commandText} [] [${ACCEPT_NEXUS_EULA_FLAG}]\` with one optional SHA and one exact consent flag.`; + await addActionComment({ github }, repoOwner, repoName, prNumber, commentUsername, commentLink, message); + break; + } + const runTests = await handleTestCommand({ core, github }, parts, "airlock tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink }); if (runTests) { command = "run-tests-airlock"; + acceptNexusEula = consentFlags.length === 1; } break; } @@ -177,6 +201,7 @@ async function getCommandFromComment({ core, context, github }) { } } logAndSetOutput(core, "skipDeployment", skipDeployment.toString()); + logAndSetOutput(core, "acceptNexusEula", acceptNexusEula.toString()); logAndSetOutput(core, "command", command); return command; } @@ -227,7 +252,7 @@ function commandHasSkipDeploymentFlag(commandParts) { } function getShaFromCommandParts(commandParts) { - return commandParts.slice(1).find(part => part !== SKIP_DEPLOYMENT_FLAG); + return commandParts.slice(1).find(part => part !== SKIP_DEPLOYMENT_FLAG && part !== ACCEPT_NEXUS_EULA_FLAG); } async function addUnsupportedSkipDeploymentFlagComment({ github }, repoOwner, repoName, prNumber, commentUser, commentLink, command) { @@ -301,9 +326,9 @@ You can use the following commands:     /test [] [skip_deployment] - build, deploy and run smoke tests on a PR     /test-extended [] [skip_deployment] - build, deploy and run smoke & extended tests on a PR     /test-extended-aad - build, deploy and run smoke & extended AAD tests on a PR -    /test-shared-services - test the deployment of shared services on a PR build +    /test-shared-services [] accept_nexus_eula - build, deploy and test shared services with explicit Nexus CE EULA acceptance     /test-backups - build, deploy and run backup tests on a PR -    /test-airlock - build, deploy and run Airlock tests on a PR +    /test-airlock [] [accept_nexus_eula] - build, deploy and run Airlock tests with optional explicit Nexus CE EULA acceptance     /test-force-approve - force approval of the PR tests (i.e. skip the deployment checks)     /test-destroy-env - delete the validation environment for a PR (e.g. to enable testing a deployment from a clean start after previous tests)     /help - show this help`; diff --git a/.github/scripts/build.test.js b/.github/scripts/build.test.js index a2b176b70d..fae4324960 100644 --- a/.github/scripts/build.test.js +++ b/.github/scripts/build.test.js @@ -476,16 +476,17 @@ describe('getCommandFromComment', () => { test(`should set command to 'run-tests-shared-services'`, async () => { const context = createCommentContext({ username: 'admin', - body: '/test-shared-services', + body: '/test-shared-services accept_nexus_eula', }); await getCommandFromComment({ core, context, github }); expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-shared-services'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('true'); }); test(`should add comment with run link`, async () => { const context = createCommentContext({ username: 'admin', - body: '/test-shared-services', + body: '/test-shared-services accept_nexus_eula', pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, }); await getCommandFromComment({ core, context, github }); @@ -497,10 +498,77 @@ describe('getCommandFromComment', () => { }); }); + test.each([ + '/test-shared-services', + '/test-shared-services accept_nexus_eula=false', + '/test-shared-services accept_nexus_eula=true', + '/test-shared-services accept_nexus_eula accept_nexus_eula=false', + '/test-shared-services accept_nexus_eula accept_nexus_eula', + '/test-shared-services\naccept_nexus_eula', + ])('does not dispatch without the explicit flag: %s', async body => { + const context = createCommentContext({ body }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + expect(mockGithubRestIssuesCreateComment).toHaveComment({ + owner: 'someOwner', repo: 'someRepo', issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, + bodyMatcher: /Nexus CE EULA acceptance is required.*No deployment or tests were started/, + }); + }); + + test.each([ + '/test-shared-services 2345678 accept_nexus_eula', + '/test-shared-services accept_nexus_eula 2345678', + ])('keeps external-PR SHA validation with the consent flag: %s', async body => { + const context = createCommentContext({ + body, pullRequestNumber: PR_NUMBER.FORK_NON_DOCS_CHANGES, authorUsername: 'non-contributor', + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-shared-services'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('true'); + }); + + test.each([ + '/test-shared-services accept_nexus_eula', + '/test-shared-services outdated123 accept_nexus_eula', + '/test-shared-services 234567 accept_nexus_eula', + ])('does not accept consent when external-PR SHA validation fails: %s', async body => { + const context = createCommentContext({ + body, pullRequestNumber: PR_NUMBER.FORK_NON_DOCS_CHANGES, authorUsername: 'non-contributor', + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + }); + + test('does not allow the consent flag to bypass collaborator permissions', async () => { + const context = createCommentContext({ + body: '/test-shared-services accept_nexus_eula', username: 'non-contributor', + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).not.toBe('true'); + }); + + test('does not accept consent for an unmergeable PR', async () => { + const context = createCommentContext({ + body: '/test-shared-services accept_nexus_eula', pullRequestNumber: PR_NUMBER.UPSTREAM_NON_MERGEABLE, + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + }); + + test('does not grant consent to another command', async () => { + const context = createCommentContext({ body: '/test-extended accept_nexus_eula' }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + }); + test(`should warn and not run if skip_deployment is supplied`, async () => { const context = createCommentContext({ username: 'admin', - body: '/test-shared-services skip_deployment', + body: '/test-shared-services skip_deployment accept_nexus_eula', }); await getCommandFromComment({ core, context, github }); expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); @@ -514,13 +582,103 @@ describe('getCommandFromComment', () => { }); describe(`for '/test-airlock'`, () => { - test(`should set command to 'run-tests-airlock'`, async () => { + test.each([ + '/test-airlock', + '/test-airlock 2345678', + '/test-airlock\naccept_nexus_eula', + ])('preserves the existing route without granting consent: %s', async body => { + const context = createCommentContext({ body }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-airlock'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + expect(outputFor(mockCoreSetOutput, 'skipDeployment')).toBe('false'); + }); + + test.each([ + '/test-airlock accept_nexus_eula', + '/test-airlock 2345678 accept_nexus_eula', + '/test-airlock accept_nexus_eula 2345678', + ])('forwards explicit consent: %s', async body => { + const context = createCommentContext({ body }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-airlock'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('true'); + expect(outputFor(mockCoreSetOutput, 'skipDeployment')).toBe('false'); + }); + + test.each([ + '/test-airlock accept_nexus_eula=true', + '/test-airlock accept_nexus_eula=false', + '/test-airlock accept_nexus_eula accept_nexus_eula=false', + '/test-airlock accept_nexus_eula accept_nexus_eula', + '/test-airlock accept_nexus_eula unexpected', + '/test-airlock accept_nexus_eula 2345678 2345678', + '/test-airlock ACCEPT_NEXUS_EULA', + '/test-airlock accept_nexus_eula 234567', + '/test-airlock accept_nexus_eula ' + 'a'.repeat(41), + ])('rejects invalid arguments without dispatching: %s', async body => { + const context = createCommentContext({ body }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + expect(mockGithubRestIssuesCreateComment).toHaveComment({ + owner: 'someOwner', repo: 'someRepo', issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, + bodyMatcher: /Invalid Airlock arguments.*No deployment or tests were started/, + }); + }); + + test.each([ + '/test-airlock 2345678 accept_nexus_eula', + '/test-airlock accept_nexus_eula 2345678', + ])('keeps reviewed SHA validation for external PRs: %s', async body => { const context = createCommentContext({ - username: 'admin', - body: '/test-airlock', + body, pullRequestNumber: PR_NUMBER.FORK_NON_DOCS_CHANGES, authorUsername: 'non-contributor', }); await getCommandFromComment({ core, context, github }); expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-airlock'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('true'); + }); + + test.each([ + '/test-airlock accept_nexus_eula', + '/test-airlock 1234567 accept_nexus_eula', + ])('does not dispatch an external PR without its reviewed SHA: %s', async body => { + const context = createCommentContext({ + body, pullRequestNumber: PR_NUMBER.FORK_NON_DOCS_CHANGES, authorUsername: 'non-contributor', + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + }); + + test('does not let consent bypass collaborator permissions', async () => { + const context = createCommentContext({ body: '/test-airlock accept_nexus_eula', username: 'non-contributor' }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).not.toBe('true'); + }); + + test('does not accept consent for an unmergeable PR', async () => { + const context = createCommentContext({ + body: '/test-airlock accept_nexus_eula', pullRequestNumber: PR_NUMBER.UPSTREAM_NON_MERGEABLE, + }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + }); + + test.each([ + '/test-airlock skip_deployment', + '/test-airlock accept_nexus_eula skip_deployment', + ])('rejects unsupported deployment reuse: %s', async body => { + const context = createCommentContext({ body }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + expect(mockGithubRestIssuesCreateComment).toHaveComment({ + owner: 'someOwner', repo: 'someRepo', issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, + bodyMatcher: /`skip_deployment` is only supported/, + }); }); }); @@ -752,7 +910,7 @@ describe('getCommandFromComment', () => { owner: 'someOwner', repo: 'someRepo', issue_number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, - bodyMatcher: /Hello!\n\nYou can use the following commands:[\s\S]*\/test-airlock/, + bodyMatcher: /Hello!\n\nYou can use the following commands:[\s\S]*\/test-airlock \[\] \[accept_nexus_eula\]/, }); }); }); diff --git a/.github/scripts/ci_environment_workflow.test.js b/.github/scripts/ci_environment_workflow.test.js index 14cf616082..d1554df810 100644 --- a/.github/scripts/ci_environment_workflow.test.js +++ b/.github/scripts/ci_environment_workflow.test.js @@ -3,7 +3,7 @@ const os = require('os'); const path = require('path'); const { execFileSync } = require('child_process'); const yaml = require('js-yaml'); -const vm = require('vm'); +const vm = require('node:vm'); const root = path.resolve(__dirname, '../..'); const workflow = name => yaml.load(fs.readFileSync(path.join(root, '.github/workflows', name), 'utf8')); @@ -154,6 +154,92 @@ describe('regional CI workflow routing', () => { beforeEach(() => { directory = fs.mkdtempSync(path.join(os.tmpdir(), 'azuretre-ci-id-')); }); afterEach(() => { fs.rmSync(directory, { recursive: true, force: true }); }); + test.each([ + ['schedule', false, '', false], + ['schedule', false, 'false', false], + ['schedule', false, 'true', true], + ['workflow_dispatch', false, '', false], + ['workflow_dispatch', true, '', true], + ['workflow_dispatch', false, 'true', true], + ['push', false, '', false], + ['push', false, 'true', true] + ])('main %s with input %s and configured consent %s selects the appropriate cases', (event, input, configured, consent) => { + const main = workflow('deploy_tre.yml'); + expect(main.on.workflow_dispatch.inputs.acceptNexusEula).toMatchObject({ type: 'boolean', default: false }); + const caller = main.jobs['run-deploy-tre-main'].with; + // These workflow expressions use JavaScript-compatible boolean operators. + const evaluate = expression => vm.runInNewContext(expression.trim().slice(3, -2), { + github: { event_name: event }, inputs: { acceptNexusEula: input }, vars: { NEXUS_EULA_ACCEPTED: configured } + }, { timeout: 1000 }); + expect(evaluate(caller.acceptNexusEula)).toBe(consent); + const selector = evaluate(caller.e2eTestsCustomSelector); + if (event === 'push') { + expect(selector).toBe('extended or extended_aad'); + } else if (consent) { + expect(selector).toBe('extended or extended_aad or shared_services or airlock'); + } else { + expect(selector).toBe('extended or extended_aad or (shared_services and not nexus) or (airlock and not nexus_required)'); + } + }); + + test('Nexus receives a separate serial job, budget and result artifact', () => { + const reusable = workflow('deploy_tre_reusable.yml'); + const plan = reusable.jobs.e2e_tests_plan; + const job = reusable.jobs.e2e_tests_custom; + expect(plan.permissions).toEqual({ contents: 'read' }); + expect(plan.environment).toBeUndefined(); + expect(plan.steps.find(step => step.id === 'select').run).toBe('python e2e_tests/ci_groups.py --plan'); + expect(job.needs).toContain('e2e_tests_plan'); + expect(job.strategy).toEqual({ 'fail-fast': false, 'max-parallel': 1, + matrix: { group: '${{ fromJSON(needs.e2e_tests_plan.outputs.groups) }}' } }); + expect(job['timeout-minutes']).toBe(300); + expect(job.steps.find(step => step.name === 'Write selected cases').env.E2E_GROUP).toBe('${{ toJSON(matrix.group) }}'); + expect(job.steps.find(step => step.name === 'Run E2E Tests').with.COMMAND).toContain('make test-e2e-ci-group'); + expect(job.steps.find(step => step.name === 'Upload Test Results').with.name).toContain('${{ matrix.group.name }}'); + expect(reusable.jobs.summary.needs).toContain('e2e_tests_plan'); + }); + + test.each([ + ['/test-airlock', false], + ['/test-airlock accept_nexus_eula', true], + ['/test-airlock 2345678 accept_nexus_eula', true], + ])('routes parsed Airlock consent into the E2E environment: %s', async (body, consent) => { + const { getCommandFromComment } = require('./build.js'); + const { createGitHubContext, PR_NUMBER } = require('./test-helpers.js'); + const { core, github, mockCoreSetOutput } = createGitHubContext(); + const context = { + runId: 11112222, + payload: { + comment: { user: { login: 'admin' }, body, html_url: 'https://example.test/comment' }, + issue: { number: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, user: { login: 'admin' } }, + repository: { full_name: 'someOwner/someRepo' }, + }, + }; + await getCommandFromComment({ core, context, github }); + const outputs = Object.fromEntries(mockCoreSetOutput.mock.calls); + const jobs = workflow('pr_comment_bot.yml').jobs; + const evaluate = (expression, values) => vm.runInNewContext(expression.trim().slice(3, -2), values, { timeout: 1000 }); + const forwarded = evaluate(jobs.pr_comment.outputs.acceptNexusEula, { steps: { check_command: { outputs } } }); + const needs = { pr_comment: { outputs: { ...outputs, acceptNexusEula: forwarded } } }; + const accepted = evaluate(jobs.run_test.with.acceptNexusEula, { needs }); + expect(accepted).toBe(consent); + expect(evaluate(jobs.run_test.with.e2eTestsCustomSelector, { needs })).toBe('airlock'); + expect(evaluate(jobs.run_test.with.skipDeployment, { needs })).toBe(false); + const runner = workflow('deploy_tre_reusable.yml').jobs.e2e_tests_custom.steps.find(step => step.name === 'Run E2E Tests'); + const command = runner.with.COMMAND.replace(/\$\{\{ inputs.acceptNexusEula \}\}/g, String(accepted)); + expect(command).toContain(`TEST_ACCEPT_NEXUS_EULA=${consent}`); + }); + + test('explicit Nexus consent reaches the reusable E2E runner as a boolean', () => { + const jobs = workflow('pr_comment_bot.yml').jobs; + expect(jobs.pr_comment.outputs.acceptNexusEula).toBe('${{ steps.check_command.outputs.acceptNexusEula }}'); + expect(jobs.run_test.with.acceptNexusEula).toBe("${{ needs.pr_comment.outputs.acceptNexusEula == 'true' }}"); + const reusable = workflow('deploy_tre_reusable.yml'); + expect(reusable.on.workflow_call.inputs.acceptNexusEula).toMatchObject({ type: 'boolean', default: false }); + const runner = reusable.jobs.e2e_tests_custom.steps.find(step => step.name === 'Run E2E Tests'); + expect(runner.with.COMMAND).toContain('TEST_ACCEPT_NEXUS_EULA=${{ inputs.acceptNexusEula }}'); + }); + test.each([ ['pr_comment_bot.yml', 'prepare_pr_env', 'run_test', 'refs/pull/5092/merge'], ['deploy_tre_branch.yml', 'prepare-not-main', 'run-deploy-tre-not-main', 'refs/heads/fix/keyvault'] diff --git a/.github/scripts/package-lock.json b/.github/scripts/package-lock.json index b28d9ccc76..f176019f64 100644 --- a/.github/scripts/package-lock.json +++ b/.github/scripts/package-lock.json @@ -1,12 +1,12 @@ { "name": "build-scripts", - "version": "1.0.4", + "version": "1.0.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "build-scripts", - "version": "1.0.4", + "version": "1.0.5", "license": "MIT", "devDependencies": { "jest": "^30.2.0", diff --git a/.github/scripts/package.json b/.github/scripts/package.json index 0452315bc7..8ce0ae331b 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -1,6 +1,6 @@ { "name": "build-scripts", - "version": "1.0.4", + "version": "1.0.5", "description": "Scripts used in the GitHub workflows", "main": "build.js", "license": "MIT", diff --git a/.github/workflows/deploy_tre.yml b/.github/workflows/deploy_tre.yml index bc41e10858..0be40232f2 100644 --- a/.github/workflows/deploy_tre.yml +++ b/.github/workflows/deploy_tre.yml @@ -16,6 +16,10 @@ on: # yamllint disable-line rule:truthy type: environment default: CICD required: true + acceptNexusEula: + description: Accept the Nexus Community Edition EULA for shared-service tests + type: boolean + default: false # The reusable deployment workflow holds the shared reference queue. # Separate caller queues can discard deployments before they reach that group. @@ -38,7 +42,9 @@ jobs: ciGitRef: ${{ github.ref }} e2eTestsCustomSelector: >- ${{ (github.event_name == 'push' && 'extended or extended_aad') - || 'extended or extended_aad or shared_services or airlock' }} + || ((inputs.acceptNexusEula || vars.NEXUS_EULA_ACCEPTED == 'true') && 'extended or extended_aad or shared_services or airlock') + || 'extended or extended_aad or (shared_services and not nexus) or (airlock and not nexus_required)' }} + acceptNexusEula: ${{ inputs.acceptNexusEula || vars.NEXUS_EULA_ACCEPTED == 'true' }} environmentName: ${{ github.event.inputs.environment || 'CICD' }} E2E_TESTS_NUMBER_PROCESSES: 1 DEVCONTAINER_TAG: "latest" diff --git a/.github/workflows/deploy_tre_branch.yml b/.github/workflows/deploy_tre_branch.yml index cd1d01cb5c..31bf151d26 100644 --- a/.github/workflows/deploy_tre_branch.yml +++ b/.github/workflows/deploy_tre_branch.yml @@ -34,6 +34,11 @@ on: # yamllint disable-line rule:truthy type: boolean default: false required: false + runCertificateTestsOnWeekends: + description: Run certificate lifecycle tests on weekends despite the rate-limit precaution + type: boolean + default: false + required: false # Register branch validation before Preparation can start. The reusable workflow # delegates this reference lock to the caller. Older workflow reruns keep their @@ -100,6 +105,7 @@ jobs: prHeadSha: ${{ github.sha }} e2eTestsCustomSelector: ${{ github.event.inputs.e2eTestsCustomSelector }} acceptNexusEula: ${{ inputs.acceptNexusEula }} + runCertificateTestsOnWeekends: ${{ inputs.runCertificateTestsOnWeekends }} skipDeployment: ${{ inputs.skipDeployment }} environmentName: ${{ github.event.inputs.environment }} E2E_TESTS_NUMBER_PROCESSES: ${{ fromJSON(github.event.inputs.e2eProcesses) }} diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 475da27073..8cb81ea4ef 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -38,6 +38,10 @@ on: # yamllint disable-line rule:truthy type: boolean default: false required: false + runCertificateTestsOnWeekends: + type: boolean + required: false + default: false environmentName: description: The name of the Github Action's environment this will deploy into type: string @@ -940,8 +944,41 @@ jobs: name: E2E Test (Smoke) Results path: "./e2e_tests/pytest_e2e_smoke.xml" + e2e_tests_plan: + name: Select E2E test groups + # Same as smoke tests: skipDeployment intentionally bypasses deploy jobs but still runs E2E tests. + if: >- + ${{ always() && inputs.e2eTestsCustomSelector != '' && + (inputs.skipDeployment || !contains(needs.*.result, 'skipped')) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') }} + runs-on: ubuntu-latest + needs: + [deploy_shared_services, register_bundles, register_user_resource_bundles] + timeout-minutes: 10 + permissions: + contents: read + outputs: + groups: ${{ steps.select.outputs.groups }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + ref: ${{ inputs.prRef }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + - name: Install collection dependencies + run: python -m pip install -r e2e_tests/unit_tests/requirements.txt + - name: Select test groups + id: select + env: + E2E_SELECTOR: ${{ inputs.e2eTestsCustomSelector }} + run: python e2e_tests/ci_groups.py --plan + e2e_tests_custom: - name: "Run E2E Tests" + name: "Run E2E Tests (${{ matrix.group.name }})" # Same as smoke tests: skipDeployment intentionally bypasses deploy jobs but still runs E2E tests. if: >- ${{ always() && inputs.e2eTestsCustomSelector != '' && @@ -952,8 +989,15 @@ jobs: runs-on: ubuntu-latest environment: ${{ inputs.environmentName }} needs: - [verify_deployment_lock, deploy_shared_services, register_bundles, register_user_resource_bundles] + [verify_deployment_lock, deploy_shared_services, register_bundles, register_user_resource_bundles, e2e_tests_plan] + # Each serial group receives its own budget. Earlier cases cannot consume + # the Nexus provisioning and cleanup window. timeout-minutes: 300 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + group: ${{ fromJSON(needs.e2e_tests_plan.outputs.groups) }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -963,10 +1007,15 @@ jobs: # then the default checkout will apply ref: ${{ inputs.prRef }} + - name: Write selected cases + env: + E2E_GROUP: ${{ toJSON(matrix.group) }} + run: printf '%s\n' "$E2E_GROUP" > e2e_tests/ci-group.json + - name: Run E2E Tests uses: ./.github/actions/devcontainer_run_command with: - COMMAND: "TEST_ACCEPT_NEXUS_EULA=${{ inputs.acceptNexusEula }} make test-e2e-custom SELECTOR='${{ inputs.e2eTestsCustomSelector }}'" + COMMAND: "TEST_ACCEPT_NEXUS_EULA=${{ inputs.acceptNexusEula }} TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS=${{ inputs.runCertificateTestsOnWeekends }} make test-e2e-ci-group" DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} @@ -990,12 +1039,12 @@ jobs: if: always() uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: E2E Test Results + name: E2E Test Results (${{ matrix.group.name }}) path: "./e2e_tests/pytest_e2e_*.xml" summary: name: Summary - needs: [e2e_tests_smoke, e2e_tests_custom] + needs: [e2e_tests_smoke, e2e_tests_plan, e2e_tests_custom] runs-on: ubuntu-latest permissions: checks: write diff --git a/.github/workflows/pr_comment_bot.yml b/.github/workflows/pr_comment_bot.yml index 64377554fd..aa3b36214b 100644 --- a/.github/workflows/pr_comment_bot.yml +++ b/.github/workflows/pr_comment_bot.yml @@ -34,6 +34,7 @@ jobs: branchCiGitRef: ${{ steps.check_command.outputs.branchCiGitRef }} ciGitRef: ${{ steps.check_command.outputs.ciGitRef }} skipDeployment: ${{ steps.check_command.outputs.skipDeployment }} + acceptNexusEula: ${{ steps.check_command.outputs.acceptNexusEula }} steps: # Ensure we have the script file for the github-script action to use - name: Checkout @@ -225,6 +226,7 @@ jobs: DEVCONTAINER_TAG: ${{ needs.prepare_pr_env.outputs.refid }} # github-script outputs strings; compare explicitly before passing to the typed workflow input. skipDeployment: ${{ needs.pr_comment.outputs.skipDeployment == 'true' }} + acceptNexusEula: ${{ needs.pr_comment.outputs.acceptNexusEula == 'true' }} secrets: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f872679036..4defc98c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Use supported review VM sizes and validate Windows tools. Forward explicit Nexus consent and bound Airlock and shared-service resource cleanup. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Wait for private certificate storage readiness, reuse trusted certificates and verify TLS for E2E identity authentication. ([#5254](https://github.com/microsoft/AzureTRE/pull/5254)) * Correct managed-identity login in bundle runtimes. Preserve failed uninstall results, allow cleanup when Porter confirms that an installation is absent, and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) * Wait for management storage, retry recognised state-read permission failures and validate Terraform outputs. Protect active and recently used validation environments from scheduled cleanup. ([#5252](https://github.com/microsoft/AzureTRE/pull/5252)) diff --git a/Makefile b/Makefile index ae51e19374..ab061901b6 100644 --- a/Makefile +++ b/Makefile @@ -554,6 +554,13 @@ test-e2e-custom: ## 🧪 Run E2E tests with custom selector (SELECTOR=) else \ python -m pytest -n "${E2E_TESTS_NUMBER_PROCESSES_DEFAULT}" -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; fi +# Description: Execute exact cases selected for one CI job. +test-e2e-ci-group: ## 🧪 Run one selected E2E CI group + . ${MAKEFILE_DIR}/devops/scripts/bootstrap_azure_env.sh \ + && . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${MAKEFILE_DIR}/e2e_tests/.env \ + && cd ${MAKEFILE_DIR}/e2e_tests \ + && python ci_groups.py --run ci-group.json + # Description: Setup the ability to debug the API and Resource Processor by configuring settings and permissions required for debugging. # Example: make setup-local-debugging setup-local-debugging: ## 🛠️ Setup local debugging diff --git a/devops/tests/test_review_vm_prerequisites.py b/devops/tests/test_review_vm_prerequisites.py new file mode 100644 index 0000000000..5af9424b95 --- /dev/null +++ b/devops/tests/test_review_vm_prerequisites.py @@ -0,0 +1,123 @@ +"""Exercise the production review-VM bootstrap with mocked Windows/network commands.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ( + ROOT / "templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1" +) +HARNESS = r""" +$ErrorActionPreference = "Stop" +$settings = $env:BOOTSTRAP_TEST_SETTINGS | ConvertFrom-Json +$source = Get-Content -Raw $env:BOOTSTRAP_SOURCE +$source = $source.Replace('${nexus_proxy_url}', 'https://nexus.example.test') +$parseErrors = $null +$tokens = $null +$ast = [System.Management.Automation.Language.Parser]::ParseInput($source, [ref]$tokens, [ref]$parseErrors) +if ($parseErrors) { throw $parseErrors[0] } +$definition = $ast.Find({ param($node) + $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Install-TreTool' +}, $true) +Invoke-Expression $definition.Extent.Text +function Wait-ForNexus { return $settings.nexus_ready } +function Configure-RProxy {} +function Configure-CondaProxy {} +function Start-Sleep {} +function Invoke-WebRequest { + param($Uri, $Method, $OutFile, [switch]$UseBasicParsing, $TimeoutSec) + if ($Method -eq 'Head' -and $settings.package_missing) { throw 'PACKAGE_MISSING' } + if ($OutFile -and $settings.download_failed) { throw 'DOWNLOAD_FAILED' } +} +function Start-Process { + param($FilePath, $ArgumentList, [switch]$Wait, [switch]$PassThru, [switch]$NoNewWindow) + return [pscustomobject]@{ ExitCode = $settings.exit_code } +} +function Get-Command { + param($Name, $ErrorAction) + if (-not $settings.cli_missing) { return [pscustomobject]@{Name = 'az'} } +} +function az { $global:LASTEXITCODE = $settings.cli_exit_code } +$NexusActionsEnabled = $true +$RequireAzureCli = $settings.required +$InstallAzureCli = $true +$ConfigureConda = $false +$AzureCliVersion = '2.88.0' +$ToolsDir = 'test-tools' +$start = $source.IndexOf('# Make sure the proxy is up') +$end = $source.IndexOf('$VsCodeSetup =') +Invoke-Expression $source.Substring($start, $end - $start) +Write-Host 'BOOTSTRAP_OK' +""" + + +@unittest.skipUnless(shutil.which("pwsh"), "PowerShell is required to execute the Windows bootstrap regression tests") +class ReviewVmPrerequisiteTests(unittest.TestCase): + def run_bootstrap(self, **overrides): + settings = { + "required": True, + "nexus_ready": True, + "package_missing": False, + "download_failed": False, + "exit_code": 0, + "cli_missing": False, + "cli_exit_code": 0, + **overrides, + } + with tempfile.TemporaryDirectory() as folder: + harness = Path(folder) / "bootstrap-test.ps1" + harness.write_text(HARNESS) + result = subprocess.run( + ["pwsh", "-NoLogo", "-NoProfile", "-File", str(harness)], + env={**os.environ, "BOOTSTRAP_SOURCE": str(SCRIPT), "BOOTSTRAP_TEST_SETTINGS": json.dumps(settings)}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + return result + + def test_missing_nexus_or_package_fails_before_install(self): + for changes, message in ( + ({"nexus_ready": False}, "reachable Nexus"), + ({"package_missing": True}, "package is unavailable"), + ): + with self.subTest(changes=changes): + result = self.run_bootstrap(**changes) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + self.assertNotIn("Installing Azure CLI", result.stdout) + + def test_required_cli_download_installer_and_execution_failures_stop_bootstrap(self): + for changes, message in ( + ({"download_failed": True}, "could not be installed"), + ({"exit_code": 1603}, "could not be installed"), + ({"cli_missing": True}, "executable was not found"), + ({"cli_exit_code": 1}, "failed its version check"), + ): + with self.subTest(changes=changes): + result = self.run_bootstrap(**changes) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, result.stderr) + self.assertNotIn("BOOTSTRAP_OK", result.stdout) + + def test_success_and_msi_reboot_required_code_are_accepted(self): + for code in (0, 3010): + with self.subTest(code=code): + result = self.run_bootstrap(exit_code=code) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("BOOTSTRAP_OK", result.stdout) + + def test_optional_cli_preserves_best_effort_behaviour(self): + for changes in ({"nexus_ready": False, "download_failed": True}, {"exit_code": 1603}): + with self.subTest(changes=changes): + result = self.run_bootstrap(required=False, **changes) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("WARNING:", result.stdout) + self.assertIn("BOOTSTRAP_OK", result.stdout) diff --git a/devops/version.txt b/devops/version.txt index 2e77e10734..c69cd12ff3 100644 --- a/devops/version.txt +++ b/devops/version.txt @@ -1 +1 @@ -__version__ = "0.6.14" +__version__ = "0.6.15" diff --git a/docs/tre-developers/github-pr-bot-commands.md b/docs/tre-developers/github-pr-bot-commands.md index 5a8b6af8ef..5fd03858ee 100644 --- a/docs/tre-developers/github-pr-bot-commands.md +++ b/docs/tre-developers/github-pr-bot-commands.md @@ -36,26 +36,67 @@ Check for changes to anything that is run during the build/deploy/test cycle, in - modifications to scripts - new python packages being installed -### `/test-extended [] [skip_deployment]` / `/test-extended-aad []`/ `/test-shared-services []` +### `/test-extended [] [skip_deployment]` / `/test-extended-aad []` -This command runs the build, deploy, and smoke & extended / shared services tests for a PR. +These commands run the build, deployment, smoke tests and selected extended tests for a PR. For PRs from maintainers (i.e. users with write access to microsoft/AzureTRE), `/test-extended` is sufficient. -If a change has been made which would affect any of the core shared services, make sure you run `/test-shared-services`. - For other PRs, the checks below should be carried out. Once satisfied that the PR is safe to run tests against, you should use `/test-extended ` where `` is the SHA for the commit that you have verified. You can use the full or short form of the SHA, but it must be at least 7 characters (GitHub UI shows 7 characters). If the PR validation environment is already deployed, add `skip_deployment` to `/test-extended` to skip the build and deployment jobs and run the smoke and extended tests against the existing environment. For PRs from forks, include both the SHA and `skip_deployment`, for example `/test-extended skip_deployment`. -The `skip_deployment` flag is not supported by `/test-extended-aad` or `/test-shared-services`. +The `skip_deployment` flag is not supported by `/test-extended-aad`. **IMPORTANT** As with `/test`, this command works on PRs from forks, and makes the deployment secrets available. Before running tests on a PR, run the same checks on the PR code as for `/test`. +### `/test-shared-services [] accept_nexus_eula` + +This command builds and deploys the PR environment, then runs smoke and shared-service tests, including the certificate and Nexus lifecycle case. +Review the PR code using the same checks as for `/test` before running it. For an external PR, include the reviewed head SHA. + +The shared-service suite creates Nexus and requires explicit acceptance of the [Sonatype Nexus Community Edition EULA](https://links.sonatype.com/products/nxrm/ce-eula). +After accepting the EULA, add the exact `accept_nexus_eula` flag on the command's first line, for example `/test-shared-services accept_nexus_eula`. +Omitting the flag prevents deployment and tests. The command does not support `skip_deployment`. + +For local execution, set `TEST_ACCEPT_NEXUS_EULA=true` before running the selected tests. +For the branch workflow, set the boolean `acceptNexusEula` input to `true` and select `shared_services`. +Without consent, the Nexus case fails before changing Nexus or certificate resources. The existing weekend certificate precaution still skips that case after consent is checked. +A skipped case does not prove the Nexus lifecycle. + +For manual runs of `deploy_tre.yml` on `main`, use its `acceptNexusEula` input. +For recurring main runs, an administrator can record consent with the repository variable `NEXUS_EULA_ACCEPTED=true`. +Either setting supplies consent. Without either setting, scheduled/manual main runs exclude the `nexus` case and retain the other shared-service tests. +Push runs retain their existing extended/AAD selection. An excluded Nexus case provides no Nexus lifecycle evidence. + +Shared-service tests allow one hour for prior-resource recovery, one hour for provisioning and one hour for each new resource's cleanup. +Nexus and certificate recovery share the first deadline. Test watchdogs include a further 30-minute margin, including failed-create cleanup. +The reusable workflow collects the selected cases and gives Nexus a separate five-hour job. +The other cases run in their own job. These jobs run sequentially against the same environment, including when a group fails. +Empty groups are omitted. Invalid or empty selections fail planning. Each job verifies the planned checkout and exact cases before execution. + +Slash commands use workflows from `main`. Before these routing changes reach `main`, validate the E2E path through `deploy_tre_branch.yml` on the reviewed upstream branch. + +### `/test-airlock [] [accept_nexus_eula]` + +This command builds and deploys the PR environment, then runs smoke tests and the `airlock` selection. +Review the code with the same checks as for `/test`. For an external PR, include the reviewed head SHA. +The command does not support `skip_deployment`. + +The review VM needs Nexus. To permit creation of the prerequisite, first accept the [Sonatype Nexus Community Edition EULA](https://links.sonatype.com/products/nxrm/ce-eula). +Then add the exact `accept_nexus_eula` flag on the command's first line, for example `/test-airlock accept_nexus_eula`. +The flag can appear before or after the SHA. Duplicate flags, `=true` or `=false` suffixes, and unknown arguments are rejected before deployment. + +Without the flag, consent remains false. The test can reuse a suitable deployed Nexus service. +If no suitable service exists, the review-VM case fails before creating Nexus. A flag on a later comment line does not grant consent. +Record selected cases, failures and skips. Independent Airlock passes do not prove the review-VM flow. + +For branch validation, use `acceptNexusEula=true`. Before the parser change reaches `main`, use the reviewed branch workflow. + ### `/test-backups []` This command runs the build, deploy, and backup tests for a PR. diff --git a/e2e_tests/airlock/request.py b/e2e_tests/airlock/request.py index a9af0f1de5..46fadad84e 100644 --- a/e2e_tests/airlock/request.py +++ b/e2e_tests/airlock/request.py @@ -4,6 +4,9 @@ import os from urllib.parse import urlparse from azure.storage.blob import BlobClient +from azure.storage.blob.aio import BlobClient as AsyncBlobClient +from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError +from uuid import uuid4 from airlock import strings from e2e_tests.helpers import get_auth_header, get_full_endpoint @@ -26,6 +29,7 @@ async def post_request(payload, endpoint, access_token, verify, assert_status): async def get_request(endpoint, access_token, verify, assert_status): + """Return JSON on success or plain text for an expected HTTP error.""" async with AsyncClient(verify=verify, timeout=TIMEOUT) as client: full_endpoint = get_full_endpoint(endpoint) auth_headers = get_auth_header(access_token) @@ -38,6 +42,10 @@ async def get_request(endpoint, access_token, verify, assert_status): assert response.status_code == assert_status + # The API's HTTP exception handler returns plain text for error responses. + if response.is_error: + return response.text + return response.json() @@ -78,6 +86,35 @@ async def delete_blob_using_sas(file_path: str, sas_url: str): client.delete_blob() +async def wait_for_draft_container_deletion(sas_url: str, *, timeout=60, interval=2): + """Require ContainerNotFound after Azure's asynchronous container deletion.""" + started = asyncio.get_running_loop().time() + parsed = urlparse(sas_url) + # A new name and overwrite avoid mistaking BlobAlreadyExists for revoked access. + blob_url = parsed._replace(path=f"{parsed.path.rstrip('/')}/revocation-probe-{uuid4().hex}").geturl() + async with AsyncBlobClient.from_blob_url(blob_url, retry_total=0, connection_timeout=5, read_timeout=10) as client: + try: + async with asyncio.timeout(timeout): + while True: + try: + await client.upload_blob(b"post-submission draft probe", overwrite=True) + except ResourceNotFoundError as error: + if getattr(error, "error_code", None) != "ContainerNotFound": + raise + LOGGER.info( + "Draft container deletion confirmed after %.1f seconds (ContainerNotFound)", + asyncio.get_running_loop().time() - started, + ) + return + except ResourceExistsError as error: + # Azure can reject writes while deletion is still in progress. + if getattr(error, "error_code", None) != "ContainerBeingDeleted": + raise + await asyncio.sleep(interval) + except TimeoutError: + raise AssertionError(f"Draft container deletion was not confirmed within {timeout} seconds") from None + + async def wait_for_status(request_status: str, workspace_owner_token, workspace_path, request_id, verify): while True: diff --git a/e2e_tests/ci_groups.py b/e2e_tests/ci_groups.py new file mode 100644 index 0000000000..9c55463871 --- /dev/null +++ b/e2e_tests/ci_groups.py @@ -0,0 +1,139 @@ +"""Partition selected E2E cases into serial CI jobs with separate time budgets.""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +E2E_ROOT = ROOT / "e2e_tests" +_GROUP = pytest.StashKey[dict]() + + +def checkout_sha(): + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + + +def validate_group(group): + if not isinstance(group, dict) or set(group) != {"name", "checkout_sha", "tests"}: + raise ValueError("Invalid E2E group request fields") + if group["name"] not in ("other", "nexus"): + raise ValueError("Unknown E2E group") + if group["checkout_sha"] != checkout_sha(): + raise ValueError("E2E group was selected from a different checkout") + tests = group["tests"] + if not isinstance(tests, list) or not tests: + raise ValueError("E2E group must contain explicit test cases") + if any( + not isinstance(nodeid, str) or not re.match(r"^(?:[a-zA-Z0-9_]+/)*test_[a-zA-Z0-9_]+\.py::", nodeid) + for nodeid in tests + ): + raise ValueError("Invalid E2E test case") + if group["name"] == "nexus" and len(tests) != 1: + raise ValueError("Each Nexus job must contain exactly one case") + if len(tests) != len(set(tests)): + raise ValueError("Duplicate E2E test cases") + return group + + +def plan(selector): + if not selector.strip(): + raise ValueError("An E2E marker selection is required") + groups = {"other": [], "nexus": []} + + class CollectGroups: + def pytest_collection_finish(self, session): + for item in session.items: + name = "nexus" if item.get_closest_marker("nexus") is not None else "other" + groups[name].append(item.nodeid) + + code = pytest.main(["--collect-only", "-q", "-o", "addopts=", "-m", selector], plugins=[CollectGroups()]) + if code != 0: + raise ValueError(f"E2E selection failed with pytest exit code {code}") + if not any(groups.values()): + raise ValueError("E2E selection contains no cases") + revision = checkout_sha() + return [ + validate_group({"name": name, "checkout_sha": revision, "tests": tests}) + for name, tests in groups.items() + if tests + ] + + +def pytest_configure(config): + # Loading this module with -p registers the guard in xdist workers too. + try: + group = validate_group(json.loads(Path(os.environ["E2E_CI_GROUP_FILE"]).read_text())) + except (ValueError, OSError, KeyError, TypeError) as error: + raise pytest.UsageError(str(error)) from error + config.stash[_GROUP] = group + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items): + group = config.stash[_GROUP] + if {item.nodeid for item in items} != set(group["tests"]): + raise pytest.UsageError("Collected E2E cases differ from the planned group") + if any((item.get_closest_marker("nexus") is not None) != (group["name"] == "nexus") for item in items): + raise pytest.UsageError("Nexus must run separately from the other E2E cases") + + +def run(filename): + group = validate_group(json.loads(filename.read_text())) + os.environ["E2E_CI_GROUP_FILE"] = str(filename) + processes = 1 if group["name"] == "nexus" else int(os.environ.get("E2E_TESTS_NUMBER_PROCESSES") or "1") + if processes < 0: + raise ValueError("E2E process count must not be negative") + args = ["-p", "e2e_tests.ci_groups", "-o", "addopts=", "--verify", os.environ.get("IS_API_SECURED", "true")] + environment = dict(os.environ) + environment["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(E2E_ROOT), environment.get("PYTHONPATH", ""))) + # Check in the controller before xdist starts workers. A worker collection + # error can otherwise lose its useful diagnostic in xdist's error handling. + validation = subprocess.run( + [sys.executable, "-m", "pytest", *args, "--collect-only", "-q", *group["tests"]], + env=environment, + capture_output=True, + text=True, + timeout=90, + check=False, + ) + if validation.returncode != 0: + print(validation.stdout, end="") + print(validation.stderr, end="", file=sys.stderr) + return validation.returncode + args.extend(["--junit-xml", f"pytest_e2e_{group['name']}.xml", *group["tests"]]) + if processes > 1: + args.extend(["-n", str(processes)]) + return int(pytest.main(args)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--plan", action="store_true") + mode.add_argument("--run", type=Path) + args = parser.parse_args() + if args.run is not None: + args.run = args.run.resolve() + os.chdir(E2E_ROOT) + sys.path[:0] = [str(ROOT), str(E2E_ROOT)] + os.environ.pop("PYTEST_ADDOPTS", None) + try: + if args.plan: + groups = plan(os.environ.get("E2E_SELECTOR", "")) + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + output.write("groups=" + json.dumps(groups, separators=(",", ":")) + "\n") + return 0 + return run(args.run) + except (ValueError, OSError, KeyError, TypeError, subprocess.TimeoutExpired) as error: + print(f"E2E group validation failed: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/e2e_tests/config.py b/e2e_tests/config.py index 125e840caa..49a85a3f36 100644 --- a/e2e_tests/config.py +++ b/e2e_tests/config.py @@ -18,6 +18,9 @@ TEST_ACCOUNT_CLIENT_SECRET: str = config("TEST_ACCOUNT_CLIENT_SECRET", default="") # Explicit operator consent is required before tests create a Nexus instance. TEST_ACCEPT_NEXUS_EULA: bool = config("TEST_ACCEPT_NEXUS_EULA", cast=bool, default=False) +TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS: bool = config( + "TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS", cast=bool, default=False +) TEST_WORKSPACE_APP_ID: str = config("TEST_WORKSPACE_APP_ID", default="") TEST_WORKSPACE_APP_SECRET: str = config("TEST_WORKSPACE_APP_SECRET", default="") TEST_WORKSPACE_APP_PLAN: str = config("WORKSPACE_APP_SERVICE_PLAN_SKU", default="") diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index cc993271bb..b0ece17c5b 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -1,18 +1,89 @@ import random +import time import pytest import asyncio from typing import Tuple import config import logging +from contextlib import AsyncExitStack, asynccontextmanager, contextmanager from resources.resource import post_resource, disable_and_delete_resource from resources.workspace import get_workspace_auth_details from resources import strings as resource_strings from helpers import get_admin_token, get_template +from e2e_tests.resources.nexus import nexus_prerequisites +from e2e_tests.timeouts import async_test_timeout, cleanup_deadline LOGGER = logging.getLogger(__name__) pytestmark = pytest.mark.asyncio(loop_scope="session") +CLEANUP_TIMEOUT_SECONDS = 60 * 60 +_TEARDOWN_ITEM = pytest.StashKey[pytest.Item]() +_ACTIVE_TIMEOUT = pytest.StashKey[tuple]() +_MANAGED_CLEANUP_USED = pytest.StashKey[bool]() + + +@pytest.hookimpl(hookwrapper=True, optionalhook=True) +def pytest_timeout_set_timer(item, settings): + outcome = yield + if outcome.excinfo is None and outcome.get_result(): + item.stash[_ACTIVE_TIMEOUT] = (settings, time.monotonic()) + + +@pytest.hookimpl(hookwrapper=True, optionalhook=True) +def pytest_timeout_cancel_timer(item): + yield + if _ACTIVE_TIMEOUT in item.stash: + del item.stash[_ACTIVE_TIMEOUT] + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_teardown(item): + item.session.stash[_TEARDOWN_ITEM] = item + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_makereport(item, call): + if call.when == "teardown" and item.stash.get(_MANAGED_CLEANUP_USED, False): + # All finalizers have finished. A restored timer must not interrupt + # traceback formatting and turn an ordinary failure into INTERNALERROR. + item.config.hook.pytest_timeout_cancel_timer(item=item) + + +@contextmanager +def pause_test_timeout_for_cleanup(request): + item = request.session.stash.get(_TEARDOWN_ITEM, None) if request is not None else None + active = item.stash.get(_ACTIVE_TIMEOUT, None) if item is not None else None + if active is None or active[0].func_only: + yield + return + settings, started = active + item.stash[_MANAGED_CLEANUP_USED] = True + remaining = settings.timeout - (time.monotonic() - started) + # A timer that already failed the body/setup needs a fresh finite budget for + # other finalizers. Otherwise preserve the unused budget, excluding cleanup. + if remaining <= 0: + remaining = settings.timeout + item.config.hook.pytest_timeout_cancel_timer(item=item) + try: + yield + finally: + item.config.hook.pytest_timeout_set_timer(item=item, settings=settings._replace(timeout=remaining)) + + +@asynccontextmanager +async def resource_cleanup_timeout(resource_path, request=None): + with pause_test_timeout_for_cleanup(request): + timeout = None + try: + async with cleanup_deadline(CLEANUP_TIMEOUT_SECONDS) as timeout: + yield + except TimeoutError as error: + if timeout is None or not timeout.expired(): + raise + message = f"Cleanup of {resource_path} exceeded {CLEANUP_TIMEOUT_SECONDS} seconds" + LOGGER.error("%s. Check the operation polling logs for its last observed state.", message) + raise TimeoutError(message) from error def pytest_addoption(parser): @@ -27,6 +98,7 @@ def verify(pytestconfig): return False +@async_test_timeout(60 * 60) async def create_or_get_test_workspace( auth_type: str, verify: bool, @@ -70,13 +142,18 @@ async def create_or_get_test_workspace( # TODO: Temp fix to solve creation of workspaces - https://github.com/microsoft/AzureTRE/issues/2986 await asyncio.sleep(random.uniform(1, 9)) workspace_path, workspace_id = await post_resource( - payload, resource_strings.API_WORKSPACES, access_token=admin_token, verify=verify + payload, + resource_strings.API_WORKSPACES, + access_token=admin_token, + verify=verify, + cleanup_failed_create=True, ) LOGGER.info(f"Workspace {workspace_id} {template_name} created") return workspace_path, workspace_id +@async_test_timeout(60 * 60) async def create_or_get_test_workpace_service( workspace_path, workspace_owner_token, pre_created_workspace_service_id, verify ): @@ -96,29 +173,32 @@ async def create_or_get_test_workpace_service( endpoint=f"/api{workspace_path}/{resource_strings.API_WORKSPACE_SERVICES}", access_token=workspace_owner_token, verify=verify, + cleanup_failed_create=True, ) return workspace_service_path, workspace_service_id -async def clean_up_test_workspace(pre_created_workspace_id: str, workspace_path: str, verify: bool): +async def clean_up_test_workspace(pre_created_workspace_id: str, workspace_path: str, verify: bool, *, request=None): # Only delete the workspace if it wasn't pre-created if pre_created_workspace_id == "": - LOGGER.info(f"Deleting workspace {pre_created_workspace_id}") - await disable_and_delete_tre_resource(workspace_path, verify) + LOGGER.info(f"Deleting workspace {workspace_path}") + async with resource_cleanup_timeout(workspace_path, request): + await disable_and_delete_tre_resource(workspace_path, verify) async def clean_up_test_workspace_service( - pre_created_workspace_service_id: str, workspace_service_path: str, workspace_id: str, verify: bool + pre_created_workspace_service_id: str, workspace_service_path: str, workspace_id: str, verify: bool, *, request=None ): if pre_created_workspace_service_id == "": - LOGGER.info(f"Deleting workspace service {pre_created_workspace_service_id}") - await disable_and_delete_ws_resource(workspace_service_path, workspace_id, verify) + LOGGER.info(f"Deleting workspace service {workspace_service_path}") + async with resource_cleanup_timeout(workspace_service_path, request): + await disable_and_delete_ws_resource(workspace_service_path, workspace_id, verify) # Session scope isn't in effect with python-xdist: https://github.com/microsoft/AzureTRE/issues/2868 @pytest.fixture(scope="session") -async def setup_test_workspace(verify) -> Tuple[str, str]: +async def setup_test_workspace(verify, request) -> Tuple[str, str]: pre_created_workspace_id = config.TEST_WORKSPACE_ID auth_type = "Manual" if config.TEST_WORKSPACE_APP_ID else "Automatic" workspace_path, workspace_id = await create_or_get_test_workspace( @@ -133,13 +213,13 @@ async def setup_test_workspace(verify) -> Tuple[str, str]: # Tear-down await clean_up_test_workspace( - pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify + pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify, request=request ) # Session scope isn't in effect with python-xdist: https://github.com/microsoft/AzureTRE/issues/2868 @pytest.fixture(scope="session") -async def setup_test_workspace_and_guacamole_service(setup_test_workspace, verify): +async def setup_test_workspace_and_guacamole_service(setup_test_workspace, verify, request): # Set up workspace_path, workspace_id = setup_test_workspace workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) @@ -155,13 +235,13 @@ async def setup_test_workspace_and_guacamole_service(setup_test_workspace, verif yield workspace_path, workspace_id, workspace_service_path, workspace_service_id await clean_up_test_workspace_service( - pre_created_workspace_service_id, workspace_service_path, workspace_id, verify + pre_created_workspace_service_id, workspace_service_path, workspace_id, verify, request=request ) # Session scope isn't in effect with python-xdist: https://github.com/microsoft/AzureTRE/issues/2868 @pytest.fixture(scope="session") -async def setup_test_aad_workspace(verify) -> Tuple[str, str, str]: +async def setup_test_aad_workspace(verify, request) -> Tuple[str, str, str]: pre_created_workspace_id = config.TEST_AAD_WORKSPACE_ID # Set up workspace_path, workspace_id = await create_or_get_test_workspace( @@ -172,7 +252,7 @@ async def setup_test_aad_workspace(verify) -> Tuple[str, str, str]: # Tear-down await clean_up_test_workspace( - pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify + pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify, request=request ) @@ -186,17 +266,32 @@ async def get_workspace_owner_token(workspace_id, verify): async def disable_and_delete_ws_resource(resource_path, workspace_id, verify): workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) - await disable_and_delete_resource(f"/api{resource_path}", workspace_owner_token, verify) + await disable_and_delete_resource(f"/api{resource_path}", workspace_owner_token, verify, allow_failed_disable=True) async def disable_and_delete_tre_resource(resource_path, verify): admin_token = await get_admin_token(verify) - await disable_and_delete_resource(f"/api{resource_path}", admin_token, verify) + await disable_and_delete_resource(f"/api{resource_path}", admin_token, verify, allow_failed_disable=True) + + +@pytest.fixture(scope="session") +async def setup_nexus_prerequisites(verify, request): + # Dependencies must outlive review VMs and their workspace finaliser. + resources = AsyncExitStack() + try: + async with asyncio.timeout(60 * 60): + await resources.enter_async_context(nexus_prerequisites(verify)) + yield + finally: + async with resource_cleanup_timeout("Nexus prerequisites", request): + await resources.aclose() # Session scope isn't in effect with python-xdist: https://github.com/microsoft/AzureTRE/issues/2868 @pytest.fixture(scope="session") -async def setup_test_airlock_import_review_workspace_and_guacamole_service(verify) -> Tuple[str, str, str, str, str]: +async def setup_test_airlock_import_review_workspace_and_guacamole_service( + setup_nexus_prerequisites, verify, request +) -> Tuple[str, str, str, str]: pre_created_workspace_id = config.TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_ID # Set up workspace_path, workspace_id = await create_or_get_test_workspace( @@ -206,22 +301,39 @@ async def setup_test_airlock_import_review_workspace_and_guacamole_service(verif pre_created_workspace_id=pre_created_workspace_id, ) - admin_token = await get_admin_token(verify=verify) - workspace_owner_token, _ = await get_workspace_auth_details( - admin_token=admin_token, workspace_id=workspace_id, verify=verify - ) + workspace_service_path = None pre_created_workspace_service_id = config.TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_SERVICE_ID + try: + admin_token = await get_admin_token(verify=verify) + workspace_owner_token, _ = await get_workspace_auth_details( + admin_token=admin_token, workspace_id=workspace_id, verify=verify + ) - workspace_service_path, workspace_service_id = await create_or_get_test_workpace_service( - workspace_path, - workspace_owner_token=workspace_owner_token, - pre_created_workspace_service_id=pre_created_workspace_service_id, - verify=verify, - ) - - yield workspace_path, workspace_id, workspace_service_path, workspace_service_id + workspace_service_path, workspace_service_id = await create_or_get_test_workpace_service( + workspace_path, + workspace_owner_token=workspace_owner_token, + pre_created_workspace_service_id=pre_created_workspace_service_id, + verify=verify, + ) - # Tear-down in a cascaded way - await clean_up_test_workspace( - pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify - ) + yield workspace_path, workspace_id, workspace_service_path, workspace_service_id + + finally: + try: + # A reused parent has no cascading cleanup. Remove only a child + # created by this fixture. Preserve any pre-existing service. + if pre_created_workspace_id and workspace_service_path is not None: + await clean_up_test_workspace_service( + pre_created_workspace_service_id, + workspace_service_path, + workspace_id, + verify, + request=request, + ) + finally: + await clean_up_test_workspace( + pre_created_workspace_id=pre_created_workspace_id, + workspace_path=workspace_path, + verify=verify, + request=request, + ) diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index 3b4a6478e7..10717ddfc3 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -5,10 +5,13 @@ markers = linux_vm: opt-in Guacamole Linux VM provisioning check requiring Nexus or explicit EULA consent to create it extended_aad shared_services + nexus: certificate and Nexus lifecycle requiring explicit EULA consent + nexus_required: tests that need an existing Nexus service or explicit EULA consent to create one backups: only backup related performance: marks tests for performance evaluation timeout: used to set test timeout with pytest-timeout airlock: only airlock related + airlock_validation: draft-container deletion and review-VM flow, including Nexus prerequisites and cleanup workspace_services asyncio_mode = auto diff --git a/e2e_tests/resources/nexus.py b/e2e_tests/resources/nexus.py new file mode 100644 index 0000000000..05acf9aa9f --- /dev/null +++ b/e2e_tests/resources/nexus.py @@ -0,0 +1,111 @@ +"""Nexus prerequisites shared by Linux and Airlock review VM tests.""" + +from contextlib import AsyncExitStack, asynccontextmanager +from uuid import uuid4 + +from packaging.version import Version + +from e2e_tests import config +from e2e_tests.helpers import get_admin_token +from e2e_tests.resources import strings +from e2e_tests.resources.resource import ( + FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS, + disable_and_delete_resource, + get_resource, + post_resource, + temporary_resource, +) +from e2e_tests.timeouts import cleanup_deadline + + +async def cleanup_certificate(resource_path, access_token, verify): + async with cleanup_deadline(FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS): + services = (await get_resource(strings.API_SHARED_SERVICES, access_token, verify))["sharedServices"] + if any(service["templateName"] == strings.NEXUS_SHARED_SERVICE for service in services): + raise RuntimeError(f"Retaining certificate dependency {resource_path}: Nexus is still present") + await disable_and_delete_resource(f"/api{resource_path}", access_token, verify, allow_failed_disable=True) + + +@asynccontextmanager +async def certificate_prerequisite(payload, endpoint, access_token, verify): + resource_path, _ = await post_resource(payload, endpoint, access_token, verify, cleanup_failed_create=True) + try: + yield resource_path + except BaseException as original_error: + try: + await cleanup_certificate(resource_path, access_token, verify) + except Exception as cleanup_error: + original_error.add_note(f"Cleanup of {resource_path} also failed: {cleanup_error!r}") + raise + else: + await cleanup_certificate(resource_path, access_token, verify) + + +@asynccontextmanager +async def nexus_prerequisites(verify): + """Reuse a suitable Nexus instance or create explicitly authorised prerequisites.""" + admin_token = await get_admin_token(verify) + services = (await get_resource(strings.API_SHARED_SERVICES, admin_token, verify))["sharedServices"] + nexus_services = [service for service in services if service["templateName"] == strings.NEXUS_SHARED_SERVICE] + assert len(nexus_services) <= 1, "This test requires at most one Nexus shared service" + if nexus_services: + nexus = nexus_services[0] + assert nexus["isEnabled"] and nexus["deploymentStatus"] in ("deployed", "updated"), ( + "Nexus must be deployed and enabled" + ) + assert Version(nexus["templateVersion"]) >= Version("3.11.0"), "Upgrade Nexus to bundle 3.11.0 before this test" + yield + return + + assert config.TEST_ACCEPT_NEXUS_EULA, ( + "This test requires Nexus. Deploy Nexus 3.11.0 first, or explicitly accept its CE EULA " + "with TEST_ACCEPT_NEXUS_EULA=true (acceptNexusEula in the branch workflow)." + ) + cert_services = [service for service in services if service["templateName"] == strings.CERTS_SHARED_SERVICE] + assert len(cert_services) <= 1, "This test requires at most one certificate shared service" + async with AsyncExitStack() as resources: + if cert_services: + certs = cert_services[0] + assert certs["isEnabled"] and certs["deploymentStatus"] in ("deployed", "updated"), ( + "Repair the existing Nexus certificate service before this test" + ) + assert certs["properties"].get("domain_prefix") == "nexus", ( + "The existing certificate service belongs to another domain. Deploy Nexus separately before this test." + ) + cert_name = certs["properties"]["cert_name"] + else: + cert_name = f"nexus-e2e-{uuid4().hex[:8]}" + await resources.enter_async_context( + certificate_prerequisite( + { + "templateName": strings.CERTS_SHARED_SERVICE, + "properties": { + "display_name": "E2E Nexus certificate", + "description": "Certificate prerequisite for E2E Nexus", + "domain_prefix": "nexus", + "cert_name": cert_name, + }, + }, + strings.API_SHARED_SERVICES, + admin_token, + verify, + ) + ) + await resources.enter_async_context( + temporary_resource( + { + "templateName": strings.NEXUS_SHARED_SERVICE, + "properties": { + "display_name": "E2E Nexus", + "description": "Nexus prerequisite for E2E workspace resources", + "ssl_cert_name": cert_name, + "accept_nexus_eula": config.TEST_ACCEPT_NEXUS_EULA, + "vm_size": "Standard_D2s_v3", + }, + }, + strings.API_SHARED_SERVICES, + admin_token, + verify, + ) + ) + yield diff --git a/e2e_tests/resources/resource.py b/e2e_tests/resources/resource.py index 3d62ae3ed6..efea3c8f91 100644 --- a/e2e_tests/resources/resource.py +++ b/e2e_tests/resources/resource.py @@ -1,6 +1,7 @@ import asyncio import logging from contextlib import asynccontextmanager +from e2e_tests.timeouts import cleanup_deadline from httpx import AsyncClient, Timeout from starlette import status from e2e_tests.helpers import assert_status, get_auth_header, get_full_endpoint @@ -10,6 +11,7 @@ LOGGER = logging.getLogger(__name__) TIMEOUT = Timeout(10, read=60) +FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS = 60 * 60 async def get_resource(endpoint, access_token, verify): @@ -24,8 +26,19 @@ async def get_resource(endpoint, access_token, verify): async def post_resource( - payload, endpoint, access_token, verify, method="POST", wait=True, etag="*", access_token_for_wait=None + payload, + endpoint, + access_token, + verify, + method="POST", + wait=True, + etag="*", + access_token_for_wait=None, + *, + cleanup_failed_create=False, ): + if cleanup_failed_create and (method != "POST" or not wait): + raise ValueError("Failed-create cleanup requires a POST with deployment polling") async with AsyncClient(verify=verify, timeout=30.0) as client: full_endpoint = get_full_endpoint(endpoint) auth_headers = get_auth_header(access_token) @@ -46,17 +59,47 @@ async def post_resource( if wait: wait_token = access_token_for_wait if access_token_for_wait is not None else access_token - await wait_for( - check_method, - client, - operation_endpoint, - wait_token, - [strings.RESOURCE_STATUS_DEPLOYMENT_FAILED, strings.RESOURCE_STATUS_UPDATING_FAILED], - ) + try: + await wait_for( + check_method, + client, + operation_endpoint, + wait_token, + [strings.RESOURCE_STATUS_DEPLOYMENT_FAILED, strings.RESOURCE_STATUS_UPDATING_FAILED], + ) + except BaseException as original_error: + if cleanup_failed_create: + cleanup = asyncio.create_task( + cleanup_accepted_create(resource_path, operation_endpoint, client, wait_token, verify) + ) + # Cancellation must not abandon a known resource before its + # fixture can register teardown. Recovery has its own deadline. + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + except Exception: + break + try: + cleanup.result() + except BaseException as cleanup_error: + original_error.add_note( + f"Cleanup of {resource_path} after {operation_endpoint} also failed: {cleanup_error!r}" + ) + raise return resource_path, resource_id +async def cleanup_accepted_create(resource_path, operation_endpoint, client, access_token, verify): + async with cleanup_deadline(FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS): + # An accepted deployment may still be running when its caller times out. + # Wait for it to finish before sending another provisioning action. + await wait_for(install_done, client, operation_endpoint, access_token, []) + await disable_and_delete_resource(f"/api{resource_path}", access_token, verify, allow_failed_disable=True) + + async def disable_and_delete_resource(endpoint, access_token, verify, *, allow_failed_disable=False): async with AsyncClient(verify=verify, timeout=TIMEOUT) as client: full_endpoint = get_full_endpoint(endpoint) @@ -71,7 +114,18 @@ async def disable_and_delete_resource(endpoint, access_token, verify, *, allow_f # The API persists isEnabled=False before provisioning. Failed installs can # fail again during disable, but deletion is safe once that operation ends. failure_states = [] if allow_failed_disable else [strings.RESOURCE_STATUS_UPDATING_FAILED] - await wait_for(patch_done, client, operation_endpoint, access_token, failure_states) + disable_state = await wait_for(patch_done, client, operation_endpoint, access_token, failure_states) + if disable_state == strings.RESOURCE_STATUS_UPDATING_FAILED: + response = await client.get(full_endpoint, headers=get_auth_header(access_token), timeout=TIMEOUT) + assert_status(response, [status.HTTP_200_OK], "Could not verify disabled resource") + resources = [ + response.json()[key] + for key in ("workspace", "workspaceService", "userResource", "sharedService") + if key in response.json() + ] + assert len(resources) == 1 and resources[0].get("isEnabled") is False, ( + "Refusing deletion after failed disable: the resource is not confirmed disabled" + ) # delete auth_headers = get_auth_header(access_token) @@ -89,41 +143,81 @@ async def disable_and_delete_resource(endpoint, access_token, verify, *, allow_f @asynccontextmanager async def temporary_resource(payload, endpoint, access_token, verify): """Create a test resource and remove it after success or failed provisioning.""" - async with AsyncClient(verify=verify, timeout=TIMEOUT) as client: - response = await client.post(get_full_endpoint(endpoint), headers=get_auth_header(access_token), json=payload) - assert_status(response, [status.HTTP_202_ACCEPTED], "The test resource could not be created") - resource_path = response.json()["operation"]["resourcePath"] + resource_path, _ = await post_resource(payload, endpoint, access_token, verify, cleanup_failed_create=True) + try: + yield resource_path + except BaseException as original_error: try: - await wait_for( - install_done, - client, - response.headers["Location"], - access_token, - [strings.RESOURCE_STATUS_DEPLOYMENT_FAILED], - ) - yield resource_path - except BaseException as original_error: - try: + async with cleanup_deadline(FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS): await disable_and_delete_resource( f"/api{resource_path}", access_token, verify, allow_failed_disable=True ) - except Exception as cleanup_error: - original_error.add_note(f"Cleanup of {resource_path} also failed: {cleanup_error!r}") - raise - else: + except Exception as cleanup_error: + original_error.add_note(f"Cleanup of {resource_path} also failed: {cleanup_error!r}") + raise + else: + async with cleanup_deadline(FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS): await disable_and_delete_resource(f"/api{resource_path}", access_token, verify) async def wait_for(func, client, operation_endpoint, access_token, failure_states: list): - done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) + done_state, message, operation_steps = "not yet observed", "", "" LOGGER.info(f"WAITING FOR OP: {operation_endpoint}") - while not done: - await asyncio.sleep(30) - + try: done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) - LOGGER.info(f"{done}, {done_state}, {message}") + while not done: + await asyncio.sleep(30) + + done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) + LOGGER.info(f"{done}, {done_state}, {message}") + except asyncio.CancelledError: + LOGGER.error( + "Stopped waiting for operation %s. Last observed state: %s; message: %s; steps: %s", + operation_endpoint, + done_state, + message, + operation_steps, + ) + raise try: assert done_state not in failure_states except Exception: LOGGER.exception(f"Failed to deploy. Status message: {message}.\n{operation_steps}") raise + return done_state + + +async def delete_owned_resource_if_present(resource_path, access_token, verify): + """Wait for active operations and remove an owned resource, unless already deleted.""" + terminal = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", + } + endpoint = get_full_endpoint(f"/api{resource_path}") + async with AsyncClient(verify=verify, timeout=TIMEOUT) as client: + while True: + response = await client.get(endpoint, headers=get_auth_header(access_token), timeout=TIMEOUT) + if response.status_code == 404: + return + assert_status(response, [status.HTTP_200_OK], "Could not inspect the owned review VM") + state = response.json()["userResource"]["deploymentStatus"] + if state == "deleted": + return + operations = await client.get( + endpoint + "/operations", headers=get_auth_header(access_token), timeout=TIMEOUT + ) + if operations.status_code == 404: + return + assert_status(operations, [status.HTTP_200_OK], "Could not inspect review VM operations") + if state in terminal and all(op["status"] in terminal for op in operations.json()["operations"]): + break + await asyncio.sleep(30) + await disable_and_delete_resource(f"/api{resource_path}", access_token, verify, allow_failed_disable=True) diff --git a/e2e_tests/test_airlock.py b/e2e_tests/test_airlock.py index 4ce8fc1c3d..d3607bff1d 100644 --- a/e2e_tests/test_airlock.py +++ b/e2e_tests/test_airlock.py @@ -1,3 +1,4 @@ +from contextlib import asynccontextmanager import os import pytest import asyncio @@ -5,12 +6,20 @@ from httpx import AsyncClient from azure.core.exceptions import ResourceNotFoundError -from airlock.request import post_request, get_request, upload_blob_using_sas, delete_blob_using_sas, wait_for_status -from resources.resource import get_resource, post_resource +from airlock.request import ( + post_request, + get_request, + upload_blob_using_sas, + delete_blob_using_sas, + wait_for_status, + wait_for_draft_container_deletion, +) +from e2e_tests.timeouts import async_test_timeout, cleanup_deadline +from resources.resource import get_resource, post_resource, delete_owned_resource_if_present from resources.workspace import get_workspace_auth_details from airlock import strings as airlock_strings from e2e_tests.conftest import get_workspace_owner_token -from helpers import get_admin_token +from helpers import assert_status, get_admin_token, get_auth_header, get_full_endpoint from e2e_tests.token_provider import TokenProvider @@ -81,17 +90,17 @@ async def submit_airlock_import_request(workspace_path: str, workspace_owner_tok @pytest.mark.timeout(30 * 60) @pytest.mark.airlock -async def test_draft_container_is_sealed_after_submit(setup_test_workspace, verify): - """A SAS handed out while the request was in Draft must stop working once it is submitted, - otherwise a researcher could alter the data after it has been scanned and reviewed.""" +@pytest.mark.airlock_validation +async def test_draft_container_is_deleted_after_submit(setup_test_workspace, verify): + """The draft SAS must stop working after asynchronous deletion completes.""" workspace_path, workspace_id = setup_test_workspace workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) _, container_url = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) - # Submission copies the data out and deletes the draft container, so the old SAS resolves to nothing. - with pytest.raises(ResourceNotFoundError): - await upload_blob_using_sas(BLOB_FILE_PATH, container_url) + # Azure permits draft access for up to 30 seconds after accepting deletion. + # The submitted copy is separate. Do not interpret a same-name conflict as denial. + await wait_for_draft_container_deletion(container_url) async def create_draft_import_request_with_file(workspace_path: str, workspace_owner_token: str, verify: bool): @@ -250,8 +259,80 @@ async def test_in_progress_data_is_not_reachable_from_the_public_internet(setup_ ) -@pytest.mark.timeout(50 * 60) +@asynccontextmanager +async def managed_review_vm(workspace_path, request_id, workspace_owner_token, review_token, verify): + resource_path, _ = await post_resource( + payload={}, + endpoint=f"/api{workspace_path}/requests/{request_id}/review-user-resource", + access_token=workspace_owner_token, + verify=verify, + method="POST", + wait=True, + access_token_for_wait=review_token, + cleanup_failed_create=True, + ) + original = None + try: + yield resource_path + except BaseException as error: + original = error + raise + finally: + + async def recover(): + async with cleanup_deadline(60 * 60): + await delete_owned_resource_if_present(resource_path, review_token, verify) + + recovery = asyncio.create_task(recover()) + cancellation = None + while not recovery.done(): + try: + await asyncio.shield(recovery) + except asyncio.CancelledError as error: + cancellation = error + except Exception: + break + try: + recovery.result() + except (Exception, asyncio.CancelledError) as error: + failure = original if original is not None else cancellation + if failure is None: + raise + failure.add_note(f"Cleanup of {resource_path} also failed: {error!r}") + if cancellation is not None and original is None: + raise cancellation + + +async def wait_for_review_vm_deletion(resource_path, access_token, verify, *, timeout_seconds=30 * 60): + """Wait for automatic deletion without sending a second delete request.""" + last_state = "not yet observed" + deadline = asyncio.timeout(timeout_seconds) + try: + async with deadline, AsyncClient(verify=verify, timeout=30) as client: + while True: + response = await client.get( + get_full_endpoint(f"/api{resource_path}"), headers=get_auth_header(access_token) + ) + if response.status_code == 404: + return + assert_status(response, [200], "Could not check review VM deletion") + last_state = response.json()["userResource"]["deploymentStatus"] + assert not last_state.endswith("_failed"), f"Review VM deletion failed: {last_state}" + if last_state == "deleted": + return + LOGGER.info("Waiting for review VM deletion: %s, %s", resource_path, last_state) + await asyncio.sleep(30) + except TimeoutError as error: + if not deadline.expired(): + raise + raise TimeoutError(f"Review VM deletion timed out. Last state: {last_state}") from error + + +@pytest.mark.timeout(150 * 60, func_only=True) @pytest.mark.airlock +@pytest.mark.airlock_validation +@pytest.mark.nexus_required +@async_test_timeout(50 * 60) async def test_airlock_review_vm_flow( setup_test_workspace, setup_test_airlock_import_review_workspace_and_guacamole_service, verify ): @@ -312,35 +393,28 @@ async def test_airlock_review_vm_flow( f"/api/workspaces/{import_review_workspace_id}/requests/{request_id}", import_workspace_owner_token, verify, 404 ) - user_resource_path, user_resource_id = await post_resource( - payload={}, - endpoint=f"/api{workspace_path}/requests/{request_id}/review-user-resource", - access_token=workspace_owner_token, - verify=verify, - method="POST", - wait=True, - access_token_for_wait=import_workspace_owner_token, # needs a different token as is created in a separate workspace - ) - - LOGGER.info(f"Airlock Review VM has been created: {user_resource_path}") - - # Approve request - LOGGER.info("Approving airlock request") - payload = {"approval": "True", "decisionExplanation": "the reason why this request was approved/rejected"} - request_result = await post_request( - payload, f"/api{workspace_path}/requests/{request_id}/review", workspace_owner_token, verify, 200 - ) - assert ( - request_result["airlockRequest"]["reviews"][0]["decisionExplanation"] - == "the reason why this request was approved/rejected" - ) - - await wait_for_status(airlock_strings.APPROVED_STATUS, workspace_owner_token, workspace_path, request_id, verify) - LOGGER.info("Airlock request has been approved") - - # Check that deletion for user resource has started - user_resource = await get_resource(f"/api{user_resource_path}", import_workspace_owner_token, verify) - assert user_resource["userResource"]["deploymentStatus"] == "updating" - LOGGER.info("Review VM has started deletion successfully") + async with managed_review_vm( + workspace_path, request_id, workspace_owner_token, import_workspace_owner_token, verify + ) as user_resource_path: + LOGGER.info(f"Airlock Review VM has been created: {user_resource_path}") + + # Approve request + LOGGER.info("Approving airlock request") + payload = {"approval": "True", "decisionExplanation": "the reason why this request was approved/rejected"} + request_result = await post_request( + payload, f"/api{workspace_path}/requests/{request_id}/review", workspace_owner_token, verify, 200 + ) + assert ( + request_result["airlockRequest"]["reviews"][0]["decisionExplanation"] + == "the reason why this request was approved/rejected" + ) + + await wait_for_status( + airlock_strings.APPROVED_STATUS, workspace_owner_token, workspace_path, request_id, verify + ) + LOGGER.info("Airlock request has been approved") + + await wait_for_review_vm_deletion(user_resource_path, import_workspace_owner_token, verify) + LOGGER.info("Review VM deletion completed") # EXPORT FLOW diff --git a/e2e_tests/test_guacamole_linuxvm.py b/e2e_tests/test_guacamole_linuxvm.py index b6f4020ec4..6b37a6353b 100644 --- a/e2e_tests/test_guacamole_linuxvm.py +++ b/e2e_tests/test_guacamole_linuxvm.py @@ -1,88 +1,16 @@ """Check that a Guacamole Linux VM finishes its Nexus-backed bootstrap.""" -from contextlib import AsyncExitStack, asynccontextmanager -from uuid import uuid4 - import pytest -from packaging.version import Version -from e2e_tests import config from e2e_tests.conftest import get_workspace_owner_token -from e2e_tests.helpers import get_admin_token from e2e_tests.resources import strings -from e2e_tests.resources.resource import get_resource, temporary_resource +from e2e_tests.resources.resource import temporary_resource +from e2e_tests.resources.nexus import nexus_prerequisites pytestmark = pytest.mark.asyncio(loop_scope="session") -@asynccontextmanager -async def linux_vm_nexus(verify): - """Reuse a suitable Nexus instance or create explicitly authorised prerequisites.""" - admin_token = await get_admin_token(verify) - services = (await get_resource(strings.API_SHARED_SERVICES, admin_token, verify))["sharedServices"] - nexus_services = [service for service in services if service["templateName"] == strings.NEXUS_SHARED_SERVICE] - assert len(nexus_services) <= 1, "This test requires at most one Nexus shared service" - if nexus_services: - nexus = nexus_services[0] - assert nexus["isEnabled"] and nexus["deploymentStatus"] in ("deployed", "updated"), ( - "Nexus must be deployed and enabled" - ) - assert Version(nexus["templateVersion"]) >= Version("3.11.0"), "Upgrade Nexus to bundle 3.11.0 before this test" - yield - return - - assert config.TEST_ACCEPT_NEXUS_EULA, ( - "This test requires Nexus. Deploy Nexus 3.11.0 first, or explicitly accept its CE EULA " - "with TEST_ACCEPT_NEXUS_EULA=true (acceptNexusEula in the branch workflow)." - ) - cert_services = [service for service in services if service["templateName"] == strings.CERTS_SHARED_SERVICE] - assert len(cert_services) <= 1, "This test requires at most one certificate shared service" - async with AsyncExitStack() as resources: - if cert_services: - certs = cert_services[0] - assert certs["isEnabled"] and certs["deploymentStatus"] in ("deployed", "updated"), ( - "Repair the existing Nexus certificate service before this test" - ) - assert certs["properties"].get("domain_prefix") == "nexus", ( - "The existing certificate service belongs to another domain. Deploy Nexus separately before this test." - ) - cert_name = certs["properties"]["cert_name"] - else: - cert_name = f"nexus-linux-e2e-{uuid4().hex[:8]}" - await resources.enter_async_context( - temporary_resource( - { - "templateName": strings.CERTS_SHARED_SERVICE, - "properties": { - "display_name": "Linux E2E Nexus certificate", - "domain_prefix": "nexus", - "cert_name": cert_name, - }, - }, - strings.API_SHARED_SERVICES, - admin_token, - verify, - ) - ) - await resources.enter_async_context( - temporary_resource( - { - "templateName": strings.NEXUS_SHARED_SERVICE, - "properties": { - "display_name": "Linux E2E Nexus", - "ssl_cert_name": cert_name, - "accept_nexus_eula": config.TEST_ACCEPT_NEXUS_EULA, - }, - }, - strings.API_SHARED_SERVICES, - admin_token, - verify, - ) - ) - yield - - @pytest.mark.linux_vm @pytest.mark.timeout(150 * 60) async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_service, verify): @@ -98,7 +26,7 @@ async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_serv }, } - async with linux_vm_nexus(verify): + async with nexus_prerequisites(verify): async with temporary_resource( payload, f"/api{workspace_service_path}/{strings.API_USER_RESOURCES}", workspace_owner_token, verify ): diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index 6e538390cf..1d8a95f03d 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -1,17 +1,88 @@ +from copy import deepcopy +import asyncio +from contextlib import AsyncExitStack, asynccontextmanager + import pytest import logging +from e2e_tests import config from e2e_tests.conftest import disable_and_delete_tre_resource from datetime import date -from resources.resource import post_resource +from resources.resource import ( + FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS, + disable_and_delete_resource, + get_resource, + post_resource, +) from helpers import get_shared_service_by_name from resources import strings from helpers import get_admin_token LOGGER = logging.getLogger(__name__) +RECOVERY_TIMEOUT_SECONDS = 60 * 60 +PROVISIONING_TIMEOUT_SECONDS = 60 * 60 +CLEANUP_TIMEOUT_SECONDS = 60 * 60 +# Recover old resources under one shared deadline. Failed creation can spend +# another cleanup deadline inside post_resource before unwinding the exit stack. +SINGLE_SERVICE_TIMEOUT_SECONDS = ( + RECOVERY_TIMEOUT_SECONDS + + PROVISIONING_TIMEOUT_SECONDS + + max(CLEANUP_TIMEOUT_SECONDS, FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS) + + 30 * 60 +) +NEXUS_TIMEOUT_SECONDS = ( + RECOVERY_TIMEOUT_SECONDS + + PROVISIONING_TIMEOUT_SECONDS + + max(2 * CLEANUP_TIMEOUT_SECONDS, FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS + CLEANUP_TIMEOUT_SECONDS) + + 30 * 60 +) +TERMINAL_STATES = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", +} + + +async def cleanup_created_shared_service(payload, resource_path, verify): + async with asyncio.timeout(CLEANUP_TIMEOUT_SECONDS): + if payload["templateName"] == strings.CERTS_SHARED_SERVICE: + admin_token = await get_admin_token(verify) + if await get_shared_service_by_name(strings.NEXUS_SHARED_SERVICE, verify, admin_token): + raise RuntimeError(f"Retaining certificate dependency {resource_path}: Nexus is still present") + await disable_and_delete_tre_resource(resource_path, verify) + + +@asynccontextmanager +async def managed_shared_service(payload, verify): + admin_token = await get_admin_token(verify) + resource_path, _ = await post_resource( + payload=payload, + endpoint=strings.API_SHARED_SERVICES, + access_token=admin_token, + verify=verify, + cleanup_failed_create=True, + ) + try: + yield resource_path + except BaseException as original_error: + try: + await cleanup_created_shared_service(payload, resource_path, verify) + except Exception as cleanup_error: + original_error.add_note(f"Cleanup of {resource_path} also failed: {cleanup_error!r}") + raise + else: + await cleanup_created_shared_service(payload, resource_path, verify) @pytest.mark.shared_services +@pytest.mark.timeout(125 * 60) async def test_patch_firewall(verify): template_name = strings.FIREWALL_SHARED_SERVICE @@ -66,17 +137,49 @@ async def test_patch_firewall(verify): admin_token = await get_admin_token(verify) shared_service_firewall = await get_shared_service_by_name(template_name, verify, admin_token) - if shared_service_firewall: - shared_service_path = f"/shared-services/{shared_service_firewall['id']}" + assert shared_service_firewall, ( + f"Firewall shared service '{template_name}' not found. Deploy it before running this test." + ) + shared_service_path = f"/shared-services/{shared_service_firewall['id']}" - await post_resource( - payload=patch_payload, - endpoint=f"/api{shared_service_path}", - access_token=admin_token, - verify=verify, - method="PATCH", - etag=shared_service_firewall["_etag"], - ) + original_properties = { + "display_name": shared_service_firewall["properties"]["display_name"], + "rule_collections": deepcopy(shared_service_firewall["properties"].get("rule_collections", [])), + } + original_error = None + try: + async with asyncio.timeout(60 * 60): + await post_resource( + payload=patch_payload, + endpoint=f"/api{shared_service_path}", + access_token=admin_token, + verify=verify, + method="PATCH", + etag=shared_service_firewall["_etag"], + ) + except BaseException as error: + original_error = error + raise + finally: + try: + async with asyncio.timeout(60 * 60): + current = (await get_resource(f"/api{shared_service_path}", admin_token, verify))["sharedService"] + current_properties = {key: current["properties"].get(key, []) for key in original_properties} + if current_properties != original_properties: + if current_properties != patch_payload["properties"]: + raise RuntimeError("Firewall properties changed concurrently; refusing to overwrite them") + await post_resource( + payload={"properties": original_properties}, + endpoint=f"/api{shared_service_path}", + access_token=admin_token, + verify=verify, + method="PATCH", + etag=current["_etag"], + ) + except Exception as restore_error: + if original_error is None: + raise + original_error.add_note(f"Firewall restoration also failed: {restore_error!r}") shared_service_templates_to_create = [ @@ -98,10 +201,10 @@ async def test_patch_firewall(verify): @pytest.mark.shared_services -@pytest.mark.timeout(50 * 60) +@pytest.mark.timeout(SINGLE_SERVICE_TIMEOUT_SECONDS) @pytest.mark.parametrize("template_name", shared_service_templates_to_create) async def test_create_shared_service(template_name, verify) -> None: - await disable_and_delete_shared_service_if_exists(template_name, verify) + await recover_previous_shared_services(verify, template_name) post_payload = { "templateName": template_name, @@ -111,29 +214,32 @@ async def test_create_shared_service(template_name, verify) -> None: }, } + if template_name == strings.ADMIN_VM_SHARED_SERVICE: + post_payload["properties"]["admin_jumpbox_vm_sku"] = "Standard_D2s_v3" + if template_name == strings.AIRLOCK_NOTIFIER_SHARED_SERVICE: post_payload["properties"].update(create_airlock_notifier_properties) - admin_token = await get_admin_token(verify) - - shared_service_path, _ = await post_resource( - payload=post_payload, - endpoint="/api/shared-services", - access_token=admin_token, - verify=verify, - ) - - await disable_and_delete_tre_resource(shared_service_path, verify) + async with AsyncExitStack() as resources: + async with asyncio.timeout(PROVISIONING_TIMEOUT_SECONDS): + await resources.enter_async_context(managed_shared_service(post_payload, verify)) @pytest.mark.shared_services -@pytest.mark.timeout(60 * 60) -@pytest.mark.skipif( - date.today().weekday() in [5, 6], reason="LetsEncrypt limits to 5 times a week. Skipping on SAT & SUN." -) +@pytest.mark.nexus +@pytest.mark.timeout(NEXUS_TIMEOUT_SECONDS) async def test_create_certs_nexus_shared_service(verify) -> None: - await disable_and_delete_shared_service_if_exists(strings.NEXUS_SHARED_SERVICE, verify) - await disable_and_delete_shared_service_if_exists(strings.CERTS_SHARED_SERVICE, verify) + if not config.TEST_ACCEPT_NEXUS_EULA: + pytest.fail( + "Nexus CE EULA acceptance is required; Nexus lifecycle coverage is unproven. " + "After accepting the EULA, set TEST_ACCEPT_NEXUS_EULA=true locally, " + "acceptNexusEula=true in the branch workflow, or add accept_nexus_eula " + "to /test-shared-services. No Nexus or certificate resources were changed." + ) + if date.today().weekday() in [5, 6] and not config.TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS: + pytest.skip("Certificate rate-limit precaution: skipping on weekends; Nexus lifecycle coverage is unproven.") + + await recover_previous_shared_services(verify, strings.NEXUS_SHARED_SERVICE, strings.CERTS_SHARED_SERVICE) cert_domain = "nexus" cert_name = "nexus-ssl" @@ -154,28 +260,24 @@ async def test_create_certs_nexus_shared_service(verify) -> None: "display_name": f"Shared service {strings.NEXUS_SHARED_SERVICE}", "description": f"{strings.NEXUS_SHARED_SERVICE} deployed via e2e tests", "ssl_cert_name": cert_name, + "vm_size": "Standard_D2s_v3", + "accept_nexus_eula": config.TEST_ACCEPT_NEXUS_EULA, }, } - admin_token = await get_admin_token(verify) - - certs_shared_service_path, _ = await post_resource( - payload=certs_post_payload, - endpoint="/api/shared-services", - access_token=admin_token, - verify=verify, - ) - - nexus_shared_service_path, _ = await post_resource( - payload=nexus_post_payload, - endpoint="/api/shared-services", - access_token=admin_token, - verify=verify, - ) + # Provisioning cancellation unwinds before cleanup starts. Each owned + # resource then receives a separate finite cleanup deadline. + async with AsyncExitStack() as resources: + async with asyncio.timeout(PROVISIONING_TIMEOUT_SECONDS): + await resources.enter_async_context(managed_shared_service(certs_post_payload, verify)) + await resources.enter_async_context(managed_shared_service(nexus_post_payload, verify)) - await disable_and_delete_tre_resource(nexus_shared_service_path, verify) - await disable_and_delete_tre_resource(certs_shared_service_path, verify) +async def recover_previous_shared_services(verify, *template_names): + # Share this budget across all prior resources, before creating new ones. + async with asyncio.timeout(RECOVERY_TIMEOUT_SECONDS): + for template_name in template_names: + await disable_and_delete_shared_service_if_exists(template_name, verify) async def disable_and_delete_shared_service_if_exists(shared_service_name, verify) -> None: @@ -184,6 +286,18 @@ async def disable_and_delete_shared_service_if_exists(shared_service_name, verif # Check that the shared service hasn't already been created shared_service = await get_shared_service_by_name(shared_service_name, verify, admin_token) if shared_service: - id = shared_service["id"] - LOGGER.info(f"Shared service {shared_service_name} already exists (id {id}), deleting it first...") - await disable_and_delete_tre_resource(f"/shared-services/{id}", verify) + properties = shared_service.get("properties", {}) + assert ( + properties.get("display_name") == f"Shared service {shared_service_name}" + and properties.get("description") == f"{shared_service_name} deployed via e2e tests" + ), "Refusing cleanup of a shared service not created by this test" + resource_id = shared_service["id"] + endpoint = f"/api/shared-services/{resource_id}" + async with asyncio.timeout(CLEANUP_TIMEOUT_SECONDS): + while True: + operations = (await get_resource(endpoint + "/operations", admin_token, verify))["operations"] + if all(operation["status"] in TERMINAL_STATES for operation in operations): + break + await asyncio.sleep(30) + LOGGER.info("Recovering prior E2E shared service %s (%s)", shared_service_name, resource_id) + await disable_and_delete_resource(endpoint, admin_token, verify, allow_failed_disable=True) diff --git a/e2e_tests/timeouts.py b/e2e_tests/timeouts.py new file mode 100644 index 0000000000..f601093ff3 --- /dev/null +++ b/e2e_tests/timeouts.py @@ -0,0 +1,40 @@ +"""Cooperative deadlines for asynchronous test bodies.""" + +import asyncio +from functools import wraps +from contextlib import asynccontextmanager +from contextvars import ContextVar + + +def async_test_timeout(seconds): + """Finish cancellation before pytest starts another test or fixture cleanup.""" + + def decorate(test): + @wraps(test) + async def bounded_test(*args, **kwargs): + async with asyncio.timeout(seconds): + return await test(*args, **kwargs) + + return bounded_test + + return decorate + + +_CLEANUP_DEADLINE = ContextVar("e2e_cleanup_deadline", default=None) + + +@asynccontextmanager +async def cleanup_deadline(seconds): + """Share one absolute deadline across nested resource finalisers.""" + loop = asyncio.get_running_loop() + outer = _CLEANUP_DEADLINE.get() + deadline = min(outer, loop.time() + seconds) if outer is not None else loop.time() + seconds + if loop.time() >= deadline: + raise TimeoutError("The shared resource cleanup deadline has expired") + token = _CLEANUP_DEADLINE.set(deadline) + timeout = asyncio.timeout_at(deadline) + try: + async with timeout: + yield timeout + finally: + _CLEANUP_DEADLINE.reset(token) diff --git a/e2e_tests/unit_tests/requirements.txt b/e2e_tests/unit_tests/requirements.txt index 8aea8546b1..3cbf870939 100644 --- a/e2e_tests/unit_tests/requirements.txt +++ b/e2e_tests/unit_tests/requirements.txt @@ -4,3 +4,5 @@ azure-identity==1.25.1 azure-cli-core==2.81.0 # Match the API validator when checking E2E request payloads against templates. jsonschema==4.25.1 +# Match the API blob client used by the Airlock request helper. +azure-storage-blob==12.27.1 diff --git a/e2e_tests/unit_tests/test_accepted_resource_cleanup.py b/e2e_tests/unit_tests/test_accepted_resource_cleanup.py new file mode 100644 index 0000000000..5eb320ea08 --- /dev/null +++ b/e2e_tests/unit_tests/test_accepted_resource_cleanup.py @@ -0,0 +1,206 @@ +"""Retain ownership of accepted resources across setup cancellation.""" + +import asyncio +from contextlib import asynccontextmanager +import json +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import AsyncClient, MockTransport, Response + +from e2e_tests import conftest as fixtures, helpers, test_airlock as airlock +from e2e_tests.resources import resource + + +@asynccontextmanager +async def template(*args): + yield Response(200, json={"properties": {}}) + + +class AcceptedResourceCleanupTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.requests = [] + self.authorization = [] + self.path = "/workspaces/new-workspace" + self.states = ["deploying", "deployed"] + self.disable_status = 202 + self.create_status = 202 + self.block_delete = False + self.poll_started = asyncio.Event() + self.delete_started = asyncio.Event() + self.release_delete = asyncio.Event() + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + self.enterContext(patch.object(fixtures, "get_admin_token", AsyncMock(return_value="admin-token"))) + self.enterContext(patch.object(fixtures, "get_template", template)) + self.enterContext(patch.object(fixtures.random, "uniform", return_value=0)) + self.enterContext(patch.object(fixtures, "post_resource", resource.post_resource)) + self.enterContext( + patch.object( + resource, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(self.handle)) + ) + ) + + async def handle(self, request): + self.requests.append((request.method, request.url.path)) + self.authorization.append(request.headers.get("Authorization")) + code = 200 + phase = request.url.path.rsplit("/", 1)[-1] + if request.method == "POST": + phase, state, code = "install", "deploying", self.create_status + elif request.method == "PATCH": + self.assertEqual(json.loads(request.content), {"isEnabled": False}) + phase, state, code = "disable", "updating", self.disable_status + elif request.method == "DELETE": + self.delete_started.set() + if self.block_delete: + await self.release_delete.wait() + phase, state = "delete", "deleting" + elif phase == "install": + self.poll_started.set() + state = self.states.pop(0) if len(self.states) > 1 else self.states[0] + else: + state = {"disable": "updated", "delete": "deleted"}[phase] + return Response( + code, + headers={"Location": f"/api/operations/{phase}"}, + json={ + "operation": { + "resourcePath": self.path, + "resourceId": self.path.rsplit("/", 1)[-1], + "status": state, + "message": "Mock operation", + "steps": [], + } + }, + ) + + async def create(self, **kwargs): + return await resource.post_resource({}, "/api/workspaces", "token", True, cleanup_failed_create=True, **kwargs) + + async def test_workspace_setup_timeout_removes_the_accepted_workspace(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): + await fixtures.create_or_get_test_workspace("Automatic", True) + self.assertEqual( + [method for method, _ in self.requests], ["POST", "GET", "GET", "PATCH", "GET", "DELETE", "GET"] + ) + self.assertEqual(self.requests[-2], ("DELETE", "/api" + self.path)) + + async def test_service_timeout_in_existing_workspace_deletes_only_the_new_service(self): + self.path = "/workspaces/existing/workspace-services/new-service" + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): + await fixtures.create_or_get_test_workpace_service("/workspaces/existing", "token", "", True) + self.assertEqual([path for method, path in self.requests if method == "DELETE"], ["/api" + self.path]) + + async def test_review_vm_timeout_finishes_child_cleanup_before_workspace_teardown(self): + self.path = "/workspaces/review/workspace-services/guacamole/user-resources/review-vm" + self.block_delete = True + events = [] + + async def post(**kwargs): + if kwargs.get("method") == "PATCH": + return "/workspaces/research", "research" + return await resource.post_resource(**kwargs) + + self.enterContext(patch.object(airlock, "post_resource", post)) + self.enterContext(patch.object(airlock, "get_workspace_owner_token", AsyncMock(return_value="research-token"))) + self.enterContext(patch.object(airlock, "get_admin_token", AsyncMock(return_value="admin-token"))) + self.enterContext( + patch.object(airlock, "get_resource", AsyncMock(return_value={"workspace": {"_etag": "etag"}})) + ) + self.enterContext( + patch.object(airlock, "get_workspace_auth_details", AsyncMock(return_value=("review-token", None))) + ) + self.enterContext( + patch.object(airlock, "submit_airlock_import_request", AsyncMock(return_value=("request", "unused"))) + ) + self.enterContext(patch.object(airlock, "get_request", AsyncMock())) + + async def run_with_teardown(): + try: + async with asyncio.timeout(0.01): + await airlock.test_airlock_review_vm_flow( + ("/workspaces/research", "research"), + ( + "/workspaces/review", + "review", + "/workspaces/review/workspace-services/guacamole", + "guacamole", + ), + True, + ) + finally: + events.append("workspace-teardown") + + task = asyncio.create_task(run_with_teardown()) + try: + await asyncio.wait_for(self.delete_started.wait(), 1) + self.assertEqual(events, []) + self.assertFalse(task.done()) + finally: + self.release_delete.set() + with self.assertRaises(TimeoutError): + await task + self.assertEqual(events, ["workspace-teardown"]) + self.assertEqual(self.requests[0], ("POST", "/api/workspaces/research/requests/request/review-user-resource")) + self.assertEqual(self.requests[-2:], [("DELETE", "/api" + self.path), ("GET", "/api/operations/delete")]) + self.assertEqual( + self.authorization, ["Bearer research-token"] + ["Bearer review-token"] * (len(self.requests) - 1) + ) + + async def test_failed_deployment_still_cleans_up_and_preserves_failure(self): + self.states = ["deployment_failed"] + with self.assertRaises(AssertionError): + await self.create() + self.assertIn(("DELETE", "/api" + self.path), self.requests) + + async def test_rejected_creation_does_not_delete_anything(self): + self.create_status = 403 + with self.assertRaises(AssertionError): + await self.create() + self.assertEqual(self.requests, [("POST", "/api/workspaces")]) + + async def test_pending_deployment_recovery_has_its_own_bound_and_records_resource(self): + self.states = ["deploying"] + with patch.object(resource, "FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS", 0.01): + with self.assertRaises(TimeoutError) as caught: + async with asyncio.timeout(0.01): + await self.create() + self.assertNotIn("PATCH", [method for method, _ in self.requests]) + note = caught.exception.__cause__.__notes__[0] + self.assertIn(self.path, note) + self.assertIn("/api/operations/install", note) + + async def test_cleanup_failure_is_attached_to_the_original_timeout(self): + self.disable_status = 403 + with self.assertRaises(TimeoutError) as caught: + async with asyncio.timeout(0.01): + await self.create() + self.assertIn("also failed", caught.exception.__cause__.__notes__[0]) + self.assertNotIn("DELETE", [method for method, _ in self.requests]) + + async def test_repeated_cancellation_waits_for_cleanup_to_finish(self): + self.block_delete = True + task = asyncio.create_task(self.create()) + await asyncio.wait_for(self.poll_started.wait(), 1) + task.cancel() + await asyncio.wait_for(self.delete_started.wait(), 1) + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + self.release_delete.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, 1) + self.assertEqual(self.requests[-1], ("GET", "/api/operations/delete")) + + async def test_cleanup_option_rejects_updates_and_unpolled_creates(self): + for options in ({"method": "PATCH"}, {"wait": False}): + with self.subTest(options=options), self.assertRaises(ValueError): + await self.create(**options) + self.assertEqual(self.requests, []) + + async def test_precreated_resources_are_returned_without_mutation(self): + await fixtures.create_or_get_test_workspace("Automatic", True, pre_created_workspace_id="existing") + await fixtures.create_or_get_test_workpace_service("/workspaces/existing", "token", "existing-service", True) + self.assertEqual(self.requests, []) diff --git a/e2e_tests/unit_tests/test_airlock_request.py b/e2e_tests/unit_tests/test_airlock_request.py new file mode 100644 index 0000000000..205a42f2f8 --- /dev/null +++ b/e2e_tests/unit_tests/test_airlock_request.py @@ -0,0 +1,111 @@ +"""Exercise Airlock response handling without deploying Azure resources.""" + +import json +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +from httpx import AsyncClient, MockTransport, Response +from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError, HttpResponseError + +from e2e_tests import helpers +from e2e_tests.airlock import request + + +class AirlockRequestTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + + async def get_response(self, response, expected_status=200): + client = AsyncClient(transport=MockTransport(lambda _: response)) + with patch.object(request, "AsyncClient", return_value=client): + return await request.get_request( + "/api/workspaces/test/requests/test/link", "test-token", True, expected_status + ) + + async def test_expected_plain_text_errors_return_without_json_decoding(self): + for status, message in ( + (400, "Airlock request is in invalid status: rejected, blocked or failed."), + (404, "Airlock request does not exist"), + ): + with self.subTest(status=status): + result = await self.get_response(Response(status, text=message), expected_status=status) + self.assertEqual(result, message) + + async def test_successful_json_response_is_decoded(self): + payload = {"airlockRequest": {"id": "test-request", "status": "draft"}} + self.assertEqual(await self.get_response(Response(200, json=payload)), payload) + + async def test_unexpected_status_still_fails(self): + for actual, expected in ((200, 400), (400, 200), (404, 400), (500, 200)): + with self.subTest(actual=actual, expected=expected): + with self.assertRaises(AssertionError): + await self.get_response(Response(actual, text="unexpected response"), expected_status=expected) + + async def test_malformed_successful_json_still_fails(self): + with self.assertRaises(json.JSONDecodeError): + await self.get_response(Response(200, text="not JSON")) + + async def test_link_response_does_not_log_sas(self): + payload = {"containerUrl": "https://storage.example.test/container?sig=test-secret-sas"} + with self.assertLogs(request.LOGGER, level="DEBUG") as logs: + self.assertEqual(await self.get_response(Response(200, json=payload)), payload) + self.assertNotIn("test-secret-sas", "\n".join(logs.output)) + + +class DraftDeletionTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.client = AsyncMock() + self.context = MagicMock() + self.context.__aenter__ = AsyncMock(return_value=self.client) + self.context.__aexit__ = AsyncMock(return_value=False) + self.factory = self.enterContext( + patch.object(request.AsyncBlobClient, "from_blob_url", return_value=self.context) + ) + self.sas = "https://storage.example.test/draft?sig=private-sas" + + def missing(self, code="ContainerNotFound"): + error = ResourceNotFoundError("Not found") + error.error_code = code + return error + + async def test_transient_writes_are_retried_until_container_not_found(self): + self.client.upload_blob.side_effect = [{"etag": "written"}, self.missing()] + await request.wait_for_draft_container_deletion(self.sas, interval=0) + self.assertEqual(self.client.upload_blob.await_count, 2) + self.assertTrue(self.client.upload_blob.call_args.kwargs["overwrite"]) + self.assertIn("/draft/revocation-probe-", self.factory.call_args.args[0]) + + async def test_container_being_deleted_is_retried_and_not_accepted_as_complete(self): + error = ResourceExistsError("Deleting") + error.error_code = "ContainerBeingDeleted" + self.client.upload_blob.side_effect = [error, self.missing()] + await request.wait_for_draft_container_deletion(self.sas, interval=0) + self.assertEqual(self.client.upload_blob.await_count, 2) + + async def test_writes_that_continue_past_deadline_fail(self): + self.client.upload_blob.return_value = {"etag": "still writable"} + with self.assertRaisesRegex(AssertionError, "not confirmed within"): + await request.wait_for_draft_container_deletion(self.sas, timeout=0.01, interval=0.001) + + async def test_conflict_authentication_and_wrong_not_found_errors_fail(self): + for error in ( + ResourceExistsError("BlobAlreadyExists"), + HttpResponseError("Forbidden"), + self.missing("BlobNotFound"), + ): + with self.subTest(error=error): + self.client.upload_blob.side_effect = error + with self.assertRaises(type(error)): + await request.wait_for_draft_container_deletion(self.sas, interval=0) + + async def test_probe_names_are_distinct_and_client_closes(self): + self.client.upload_blob.side_effect = self.missing() + await request.wait_for_draft_container_deletion(self.sas) + first = self.factory.call_args.args[0] + await request.wait_for_draft_container_deletion(self.sas) + self.assertNotEqual(first, self.factory.call_args.args[0]) + self.assertEqual(self.context.__aexit__.await_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e_tests/unit_tests/test_ci_groups.py b/e2e_tests/unit_tests/test_ci_groups.py new file mode 100644 index 0000000000..fe15a79768 --- /dev/null +++ b/e2e_tests/unit_tests/test_ci_groups.py @@ -0,0 +1,134 @@ +"""Exercise CI group collection, isolation and execution without Azure.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from e2e_tests import ci_groups + + +SOURCE = """ +from pathlib import Path +import pytest + +@pytest.mark.shared_services +def test_other(): + Path("other-ran").touch() + +@pytest.mark.shared_services +@pytest.mark.nexus +def test_nexus(): + Path("nexus-ran").touch() +""" + + +class CIGroupsTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.folder = Path(self.temporary.name) + (self.folder / "test_cases.py").write_text(SOURCE) + (self.folder / "conftest.py").write_text( + 'def pytest_addoption(parser):\n parser.addoption("--verify", default="true")\n' + ) + (self.folder / "pytest.ini").write_text("[pytest]\nmarkers =\n shared_services\n nexus\n") + self.output = self.folder / "output" + + def invoke(self, *args, extra=None): + environment = dict( + os.environ, + PYTHONPATH=os.pathsep.join((str(ci_groups.ROOT), str(ci_groups.E2E_ROOT))), + GITHUB_OUTPUT=str(self.output), + E2E_TESTS_NUMBER_PROCESSES="1", + ) + environment.pop("E2E_CI_GROUP_FILE", None) + environment.update(extra or {}) + script = ( + "from pathlib import Path; from e2e_tests import ci_groups; " + "ci_groups.E2E_ROOT = Path.cwd(); raise SystemExit(ci_groups.main())" + ) + return subprocess.run( + [sys.executable, "-c", script, *args], + cwd=self.folder, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + def plan(self, selector="shared_services"): + result = self.invoke("--plan", extra={"E2E_SELECTOR": selector}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return json.loads(self.output.read_text().split("groups=")[-1]) + + def run_group(self, group, **kwargs): + filename = self.folder / "request.json" + filename.write_text(json.dumps(group)) + return self.invoke("--run", str(filename), **kwargs) + + def test_selected_cases_are_partitioned_without_loss_or_overlap(self): + groups = self.plan() + self.assertEqual([g["name"] for g in groups], ["other", "nexus"]) + self.assertEqual(groups[0]["tests"], ["test_cases.py::test_other"]) + self.assertEqual(groups[1]["tests"], ["test_cases.py::test_nexus"]) + self.assertFalse((self.folder / "other-ran").exists()) + self.assertFalse((self.folder / "nexus-ran").exists()) + + def test_empty_groups_are_omitted_and_empty_selection_fails(self): + self.assertEqual([g["name"] for g in self.plan("shared_services and not nexus")], ["other"]) + self.assertEqual([g["name"] for g in self.plan("nexus")], ["nexus"]) + for selector in ("", "missing", "shared_services and ("): + with self.subTest(selector=selector): + self.output.unlink(missing_ok=True) + result = self.invoke("--plan", extra={"E2E_SELECTOR": selector}) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.output.exists()) + + def test_extra_nexus_cases_cannot_share_the_same_job_budget(self): + with (self.folder / "test_cases.py").open("a") as source: + source.write("\n@pytest.mark.nexus\ndef test_more_nexus():\n pass\n") + result = self.invoke("--plan", extra={"E2E_SELECTOR": "nexus"}) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("exactly one case", result.stderr) + self.assertFalse(self.output.exists()) + + def test_each_job_executes_only_its_cases_with_separate_reports(self): + for group in self.plan(): + result = self.run_group(group, extra={"PYTEST_ADDOPTS": "-k missing", "E2E_TESTS_NUMBER_PROCESSES": "2"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue((self.folder / f"{group['name']}-ran").exists()) + self.assertTrue((self.folder / f"pytest_e2e_{group['name']}.xml").exists()) + if group["name"] == "other": + self.assertFalse((self.folder / "nexus-ran").exists()) + + def test_nexus_cannot_be_hidden_in_the_other_job_including_xdist(self): + group = self.plan()[1] + group["name"] = "other" + for processes in ("1", "2"): + with self.subTest(processes=processes): + result = self.run_group(group, extra={"E2E_TESTS_NUMBER_PROCESSES": processes}) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("Nexus must run separately", result.stdout + result.stderr) + self.assertFalse((self.folder / "nexus-ran").exists()) + + def test_stale_empty_and_malformed_requests_fail_before_execution(self): + original = self.plan()[0] + invalid = [ + None, + {}, + {**original, "checkout_sha": "0" * 40}, + {**original, "tests": []}, + {**original, "tests": ["--help"]}, + {**original, "tests": ["../test_cases.py::test_other"]}, + {**original, "tests": original["tests"] * 2}, + ] + for group in invalid: + with self.subTest(group=group): + result = self.run_group(group) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertFalse((self.folder / "other-ran").exists()) diff --git a/e2e_tests/unit_tests/test_disable_deadline.py b/e2e_tests/unit_tests/test_disable_deadline.py new file mode 100644 index 0000000000..a008925182 --- /dev/null +++ b/e2e_tests/unit_tests/test_disable_deadline.py @@ -0,0 +1,60 @@ +"""Exercise the accepted-disable boundary under the real fixture deadline.""" + +import asyncio +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import AsyncClient, MockTransport, Response +from e2e_tests import conftest as fixtures, helpers +from e2e_tests.resources import resource + + +class DisableDeadlineTests(unittest.IsolatedAsyncioTestCase): + async def run_cleanup(self, stall): + self.requests = [] + + async def handle(request): + self.requests.append((request.method, request.url.path)) + if request.method == "PATCH": + phase, state, code = "disable", "updating", 202 + elif request.method == "DELETE": + phase, state, code = "delete", "deleting", 200 + else: + phase, code = request.url.path.rsplit("/", 1)[-1], 200 + if phase == "disable": + await asyncio.sleep(1 if stall else 0.001) + state = {"disable": "updated", "delete": "deleted"}[phase] + return Response( + code, + headers={"Location": f"/api/operations/{phase}"}, + json={ + "operation": { + "resourceId": "owned-workspace", + "status": state, + "message": "test", + "steps": [], + } + }, + ) + + with ( + patch.object(helpers.config, "TRE_URL", "https://tre.example.test"), + patch.object(resource, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(handle))), + patch.object(fixtures, "disable_and_delete_resource", resource.disable_and_delete_resource), + patch.object(fixtures, "get_admin_token", AsyncMock(return_value="token")), + patch.object(fixtures, "CLEANUP_TIMEOUT_SECONDS", 0.03), + ): + await fixtures.clean_up_test_workspace("", "/workspaces/owned-workspace", True) + + async def test_accepted_disable_reaches_terminal_state_before_delete(self): + await self.run_cleanup(False) + self.assertEqual([method for method, _ in self.requests], ["PATCH", "GET", "DELETE", "GET"]) + + async def test_stalled_disable_has_bounded_failure_and_does_not_send_conflicting_delete(self): + with self.assertLogs(resource.LOGGER, level="ERROR") as logs: + with self.assertRaisesRegex(TimeoutError, "Cleanup of /workspaces/owned-workspace exceeded"): + await asyncio.wait_for(self.run_cleanup(True), 0.5) + self.assertEqual([method for method, _ in self.requests], ["PATCH", "GET"]) + self.assertIn("/api/operations/disable", "\n".join(logs.output)) + await asyncio.sleep(0.01) + self.assertEqual([method for method, _ in self.requests], ["PATCH", "GET"]) diff --git a/e2e_tests/unit_tests/test_firewall_patch.py b/e2e_tests/unit_tests/test_firewall_patch.py new file mode 100644 index 0000000000..a2977e72e9 --- /dev/null +++ b/e2e_tests/unit_tests/test_firewall_patch.py @@ -0,0 +1,86 @@ +"""Check firewall mutation, restoration and concurrent-change protection.""" + +import asyncio +from copy import deepcopy +import unittest +from unittest.mock import AsyncMock, patch + +from e2e_tests import test_shared_services as shared + + +class FirewallPatchTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.original = { + "display_name": "Core firewall", + "rule_collections": [{"name": "original-rule"}], + "private_ip": "10.0.0.1", + } + self.firewall = {"id": "firewall-id", "_etag": "original-etag", "properties": deepcopy(self.original)} + self.enterContext(patch.object(shared, "get_admin_token", AsyncMock(return_value="token"))) + self.find = self.enterContext( + patch.object(shared, "get_shared_service_by_name", AsyncMock(return_value=self.firewall)) + ) + self.read = self.enterContext( + patch.object(shared, "get_resource", AsyncMock(return_value={"sharedService": self.firewall})) + ) + self.failure = None + self.restore_failure = None + self.concurrent = False + self.calls = [] + + async def update(**kwargs): + self.calls.append(deepcopy(kwargs)) + if len(self.calls) == 2 and self.restore_failure: + raise self.restore_failure + self.firewall["properties"].update(deepcopy(kwargs["payload"]["properties"])) + self.firewall["_etag"] = "updated-etag" + if len(self.calls) == 1: + if self.concurrent: + self.firewall["properties"]["display_name"] = "Administrator edit" + if self.failure: + raise self.failure + + self.post = self.enterContext(patch.object(shared, "post_resource", AsyncMock(side_effect=update))) + + async def test_missing_firewall_fails_without_patching(self): + self.find.return_value = None + with self.assertRaisesRegex(AssertionError, "Firewall shared service.*not found"): + await shared.test_patch_firewall(True) + self.post.assert_not_awaited() + + async def test_success_restores_original_properties_using_fresh_etag(self): + await shared.test_patch_firewall(True) + self.assertEqual(self.firewall["properties"], self.original) + self.assertEqual([call["etag"] for call in self.calls], ["original-etag", "updated-etag"]) + self.assertEqual(len(self.calls[0]["payload"]["properties"]["rule_collections"]), 3) + self.assertNotIn("private_ip", self.calls[1]["payload"]["properties"]) + self.read.assert_awaited_once_with("/api/shared-services/firewall-id", "token", True) + + async def test_failed_or_cancelled_patch_restores_original_properties(self): + for error in (RuntimeError("PATCH_FAILED"), asyncio.CancelledError()): + with self.subTest(error=type(error)): + self.calls.clear() + self.failure = error + with self.assertRaises(type(error)) as caught: + await shared.test_patch_firewall(True) + self.assertIs(caught.exception, error) + self.assertEqual(self.firewall["properties"], self.original) + + async def test_restore_failure_fails_an_otherwise_successful_test(self): + self.restore_failure = RuntimeError("RESTORE_FAILED") + with self.assertRaisesRegex(RuntimeError, "RESTORE_FAILED"): + await shared.test_patch_firewall(True) + + async def test_restore_failure_preserves_the_original_patch_error(self): + self.failure = RuntimeError("PATCH_FAILED") + self.restore_failure = RuntimeError("RESTORE_FAILED") + with self.assertRaisesRegex(RuntimeError, "PATCH_FAILED") as caught: + await shared.test_patch_firewall(True) + self.assertIn("RESTORE_FAILED", caught.exception.__notes__[0]) + + async def test_concurrent_edits_are_not_overwritten(self): + self.concurrent = True + with self.assertRaisesRegex(RuntimeError, "changed concurrently"): + await shared.test_patch_firewall(True) + self.assertEqual(self.firewall["properties"]["display_name"], "Administrator edit") + self.assertEqual(len(self.calls), 1) diff --git a/e2e_tests/unit_tests/test_fixture_cleanup.py b/e2e_tests/unit_tests/test_fixture_cleanup.py new file mode 100644 index 0000000000..6669fdb678 --- /dev/null +++ b/e2e_tests/unit_tests/test_fixture_cleanup.py @@ -0,0 +1,432 @@ +"""Check real session fixtures against the pinned pytest timeout plugins.""" + +import asyncio +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest +from unittest.mock import AsyncMock +import xml.etree.ElementTree as ET + +from e2e_tests.resources import resource + + +REPO_ROOT = Path(__file__).resolve().parents[2] +FIXTURE_STUBS = """ +import asyncio +import json +import time +from pathlib import Path +import pytest +from e2e_tests import conftest as fixtures +from contextlib import asynccontextmanager + +@asynccontextmanager +async def nexus_prerequisites(verify): + if settings.get("record_nexus"): + event(["nexus", "ready"]) + try: + yield + finally: + if settings.get("record_nexus"): + event(["nexus", "delete"]) + +fixtures.nexus_prerequisites = nexus_prerequisites + +settings = json.loads(Path("settings.json").read_text()) +fixtures.CLEANUP_TIMEOUT_SECONDS = settings["budget"] + +def event(value): + with Path("events.jsonl").open("a") as log: + log.write(json.dumps(value) + "\\n") + +async def create_workspace(**kwargs): + name = "review" if "review" in kwargs.get("template_name", "") else "research" + return f"/workspaces/{name}", name + +async def token(*args, **kwargs): + return "offline-token" + +async def auth_details(*args, **kwargs): + return "offline-workspace-token", {} + +async def create_service(*args, **kwargs): + if settings.get("fail_service_setup"): + raise RuntimeError("SERVICE_SETUP_FAILED") + return "/workspaces/review/workspace-services/service", "service" + +async def delete_resource(resource_path, *args): + name = resource_path.rsplit("/", 1)[1] + event([name, "start"]) + try: + if name == settings.get("failure_resource"): + raise RuntimeError("DELETE_FAILED") + await asyncio.sleep(settings["durations"].get(name, 0.01)) + event([name, "complete"]) + except asyncio.CancelledError: + event([name, "cancel"]) + # The next fixture must wait for asynchronous cancellation to finish. + await asyncio.sleep(0.03) + event([name, "cancel_complete"]) + raise + finally: + event([name, "finish"]) + +fixtures.create_or_get_test_workspace = create_workspace +fixtures.get_admin_token = token +fixtures.get_workspace_auth_details = auth_details +fixtures.create_or_get_test_workpace_service = create_service +fixtures.disable_and_delete_tre_resource = delete_resource +fixtures.disable_and_delete_ws_resource = delete_resource +fixtures.config.TEST_WORKSPACE_ID = "" +fixtures.config.TEST_WORKSPACE_SERVICE_ID = "" +fixtures.config.TEST_AAD_WORKSPACE_ID = "" +fixtures.config.TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_ID = "" +fixtures.config.TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_SERVICE_ID = "" + +# Register the actual fixtures and hooks, with only Azure calls replaced. +setup_nexus_prerequisites = fixtures.setup_nexus_prerequisites +setup_test_workspace = fixtures.setup_test_workspace +setup_test_workspace_and_guacamole_service = fixtures.setup_test_workspace_and_guacamole_service +setup_test_aad_workspace = fixtures.setup_test_aad_workspace +setup_test_airlock_import_review_workspace_and_guacamole_service = ( + fixtures.setup_test_airlock_import_review_workspace_and_guacamole_service +) +pytest_timeout_set_timer = fixtures.pytest_timeout_set_timer +pytest_timeout_cancel_timer = fixtures.pytest_timeout_cancel_timer +pytest_runtest_teardown = fixtures.pytest_runtest_teardown +pytest_runtest_makereport = fixtures.pytest_runtest_makereport + +class SlowReportPlugin: + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(self, item, call): + yield + if call.when == "teardown" and settings.get("slow_report"): + time.sleep(0.2) + +def pytest_configure(config): + config.pluginmanager.register(SlowReportPlugin()) + +@pytest.fixture(scope="session") +def verify(): + return True +""" +RESOURCE_TEST = """ +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.timeout(0.1) +async def test_resources( + setup_test_workspace, + setup_test_airlock_import_review_workspace_and_guacamole_service, +): + assert setup_test_workspace[1] == "research" + assert setup_test_airlock_import_review_workspace_and_guacamole_service[1] == "review" +""" +UNRELATED_TEST = """ +@pytest.mark.timeout(0.1) +def test_unrelated(): + assert True +""" + + +class FixtureCleanupIntegrationTests(unittest.TestCase): + def run_pytest( + self, + test_source, + *, + budget=1, + durations=None, + failure_resource=None, + record_nexus=False, + fail_service_setup=False, + slow_report=False, + ): + settings = { + "budget": budget, + "durations": durations or {"review": 0.2, "research": 0.2}, + "failure_resource": failure_resource, + "record_nexus": record_nexus, + "fail_service_setup": fail_service_setup, + "slow_report": slow_report, + } + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + (folder / "conftest.py").write_text(textwrap.dedent(FIXTURE_STUBS)) + (folder / "test_cases.py").write_text( + "import time\nimport pytest\nimport asyncio\nfrom conftest import event\nfrom e2e_tests.timeouts import async_test_timeout\n" + + test_source + ) + (folder / "settings.json").write_text(json.dumps(settings)) + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join((str(REPO_ROOT), str(REPO_ROOT / "e2e_tests"))) + environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + environment.pop("PYTEST_ADDOPTS", None) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-p", + "pytest_asyncio.plugin", + "-p", + "pytest_timeout", + "-o", + "asyncio_mode=auto", + "-o", + "asyncio_default_fixture_loop_scope=session", + "--junitxml=results.xml", + "--tb=short", + "-q", + "test_cases.py", + ], + cwd=directory, + env=environment, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + self.assertTrue((folder / "results.xml").exists(), result.stdout + result.stderr) + report = ET.parse(folder / "results.xml").getroot() + events_file = folder / "events.jsonl" + events = [json.loads(line) for line in events_file.read_text().splitlines()] if events_file.exists() else [] + return result, report, events + + def assert_sequential_cleanup(self, events): + self.assertEqual( + events, + [ + ["review", "start"], + ["review", "complete"], + ["review", "finish"], + ["research", "start"], + ["research", "complete"], + ["research", "finish"], + ], + ) + + def test_slow_cleanup_does_not_inherit_last_test_budget_in_either_order(self): + for source in (RESOURCE_TEST + UNRELATED_TEST, UNRELATED_TEST + RESOURCE_TEST): + with self.subTest(order=source): + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//testcase")), 2) + self.assert_sequential_cleanup(events) + + def test_nexus_outlives_review_workspace_cleanup(self): + result, _, events = self.run_pytest(RESOURCE_TEST, record_nexus=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertLess(events.index(["nexus", "ready"]), events.index(["review", "start"])) + self.assertLess(events.index(["review", "complete"]), events.index(["nexus", "delete"])) + + def test_restored_timer_does_not_interrupt_teardown_error_reporting(self): + result, report, events = self.run_pytest(RESOURCE_TEST, failure_resource="review", slow_report=True) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("DELETE_FAILED", result.stdout) + self.assertNotIn("INTERNALERROR", result.stdout) + self.assertIn(["research", "complete"], events) + + def test_service_setup_failure_still_cleans_up_review_workspace_before_nexus(self): + source = RESOURCE_TEST.replace("@pytest.mark.timeout(0.1)", "@pytest.mark.timeout(1, func_only=True)") + result, report, events = self.run_pytest(source, record_nexus=True, fail_service_setup=True) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("SERVICE_SETUP_FAILED", result.stdout) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertLess(events.index(["review", "complete"]), events.index(["nexus", "delete"])) + self.assertIn(["research", "complete"], events) + + def test_cleanup_timeout_finishes_cancellation_before_next_fixture(self): + # Keep the independent cleanup deadline short, but leave enough time + # for pytest to format its expected error on slower CI runners. + source = (RESOURCE_TEST + UNRELATED_TEST).replace("@pytest.mark.timeout(0.1)", "@pytest.mark.timeout(1)") + result, report, events = self.run_pytest( + source, + budget=0.05, + durations={"review": 1, "research": 0.01}, + ) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("Cleanup of /workspaces/review exceeded 0.05 seconds", result.stdout) + self.assertEqual( + events, + [ + ["review", "start"], + ["review", "cancel"], + ["review", "cancel_complete"], + ["review", "finish"], + ["research", "start"], + ["research", "complete"], + ["research", "finish"], + ], + ) + + def test_body_and_cleanup_failures_are_both_reported(self): + source = ( + RESOURCE_TEST + + """ +@pytest.mark.timeout(0.1) +def test_failure(): + assert False, "BODY_FAILED" +""" + ) + result, report, events = self.run_pytest(source, failure_resource="review") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//failure")), 1) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("BODY_FAILED", result.stdout) + self.assertIn("DELETE_FAILED", result.stdout) + self.assertIn(["research", "complete"], events) + + def test_body_timeout_remains_failure_and_session_cleanup_finishes(self): + source = ( + RESOURCE_TEST + + """ +@pytest.mark.timeout(0.05) +def test_body_timeout(): + time.sleep(1) +""" + ) + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//failure")), 1) + self.assertEqual(len(report.findall(".//error")), 0) + self.assertIn("Timeout >0.05s", result.stdout) + self.assert_sequential_cleanup(events) + + def test_async_body_timeout_cancels_polling_before_next_test_and_cleanup(self): + source = ( + RESOURCE_TEST + + """ +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.timeout(1) +@async_test_timeout(0.02) +async def test_async_timeout(): + try: + await asyncio.sleep(10) + event(["body", "leaked"]) + finally: + await asyncio.sleep(0.03) + event(["body", "cancel_complete"]) + +@pytest.mark.asyncio(loop_scope="session") +async def test_after_timeout(): + event(["next", "start"]) + await asyncio.sleep(0.05) +""" + ) + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//failure")), 1) + self.assertEqual(len(report.findall(".//error")), 0) + self.assertIn("TimeoutError", result.stdout) + self.assertEqual(events[:2], [["body", "cancel_complete"], ["next", "start"]]) + self.assert_sequential_cleanup(events[2:]) + self.assertNotIn("Task exception was never retrieved", result.stdout + result.stderr) + + def test_setup_timeout_remains_error_and_session_cleanup_finishes(self): + source = ( + RESOURCE_TEST + + """ +@pytest.fixture +def slow_setup(): + time.sleep(1) + +@pytest.mark.timeout(0.05) +def test_setup_timeout(slow_setup): + assert False, "body must not run" +""" + ) + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("Timeout >0.05s", result.stdout) + self.assert_sequential_cleanup(events) + + def test_unmanaged_fixture_teardown_keeps_test_timeout(self): + source = """ +@pytest.fixture +def unmanaged_fixture(): + yield + time.sleep(1) + +@pytest.mark.timeout(0.05) +def test_unmanaged(unmanaged_fixture): + assert True +""" + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("Timeout >0.05s", result.stdout) + self.assertEqual(events, []) + + def test_workspace_service_cleanup_has_independent_timeout(self): + source = """ +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.timeout(1) +async def test_service(setup_test_workspace_and_guacamole_service): + assert setup_test_workspace_and_guacamole_service[3] == "service" +""" + result, report, events = self.run_pytest( + source, + budget=0.05, + durations={"service": 1, "research": 0.01}, + ) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("Cleanup of /workspaces/review/workspace-services/service exceeded 0.05 seconds", result.stdout) + self.assertLess(events.index(["service", "cancel_complete"]), events.index(["research", "start"])) + + def test_unmanaged_function_finalizer_keeps_timeout_before_managed_cleanup(self): + source = ( + RESOURCE_TEST + + """ +@pytest.fixture +def unmanaged_fixture(): + yield + event(["unmanaged", "start"]) + time.sleep(1) + event(["unmanaged", "leaked"]) + +@pytest.mark.timeout(0.05) +def test_last(unmanaged_fixture): + assert True +""" + ) + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("Timeout >0.05s", result.stdout) + self.assertEqual(events[0], ["unmanaged", "start"]) + self.assert_sequential_cleanup(events[1:]) + + def test_unmanaged_session_finalizer_keeps_timeout_after_managed_cleanup(self): + source = """ +@pytest.fixture(scope="session") +def unmanaged_fixture(): + yield + event(["unmanaged", "start"]) + time.sleep(1) + event(["unmanaged", "leaked"]) +""" + RESOURCE_TEST.replace(" setup_test_workspace,", " unmanaged_fixture,\n setup_test_workspace,") + result, report, events = self.run_pytest(source) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(len(report.findall(".//error")), 1) + self.assertIn("Timeout >", result.stdout) + self.assert_sequential_cleanup(events[:-1]) + self.assertEqual(events[-1], ["unmanaged", "start"]) + + +class CleanupPollingDiagnosticsTests(unittest.IsolatedAsyncioTestCase): + async def test_cancellation_records_last_observed_operation_state(self): + check = AsyncMock(return_value=(False, "pipeline_running", "Waiting for resource", "Step 1")) + with self.assertLogs(resource.LOGGER, level="ERROR") as logs: + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): + await resource.wait_for(check, None, "/api/operations/cleanup", "token", []) + self.assertIn("/api/operations/cleanup", logs.output[0]) + self.assertIn("pipeline_running", logs.output[0]) + self.assertIn("Waiting for resource", logs.output[0]) diff --git a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py index 17831a41fc..c174446ba4 100644 --- a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py +++ b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py @@ -10,14 +10,14 @@ from e2e_tests import helpers from e2e_tests import test_guacamole_linuxvm as linuxvm -from e2e_tests.resources import resource, strings +from e2e_tests.resources import nexus, resource, strings class LinuxVmCleanupTests(unittest.IsolatedAsyncioTestCase): def setUp(self): self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) self.enterContext(patch.object(linuxvm, "get_workspace_owner_token", new=AsyncMock(return_value="token"))) - self.enterContext(patch.object(linuxvm, "get_admin_token", new=AsyncMock(return_value="admin-token"))) + self.enterContext(patch.object(nexus, "get_admin_token", new=AsyncMock(return_value="admin-token"))) self.fixture = ("/workspaces/ws", "ws", "/workspaces/ws/workspace-services/service", "service") self.resource_path = self.fixture[2] + "/user-resources/linux" self.requests = [] @@ -35,8 +35,12 @@ def mock_api( disable_status=202, disable_outcome=strings.RESOURCE_STATUS_UPDATED, delete_outcome=strings.RESOURCE_STATUS_DELETED, + resource_disabled=True, ): + install_polls = 0 + def handle(request): + nonlocal install_polls if request.url.path == "/api/shared-services": return Response( 200, @@ -52,6 +56,8 @@ def handle(request): }, ) self.requests.append(request) + if request.method == "GET" and request.url.path == f"/api{self.resource_path}": + return Response(200, json={"userResource": {"isEnabled": not resource_disabled}}) phase = request.url.path.rsplit("/", 1)[-1] response_status = 200 if request.method == "POST": @@ -65,6 +71,10 @@ def handle(request): else: self.assertEqual(request.method, "GET") state = {"install": outcome, "disable": disable_outcome, "delete": delete_outcome}[phase] + if phase == "install": + install_polls += 1 + if isinstance(state, Exception) and install_polls > 1: + state = strings.RESOURCE_STATUS_DEPLOYMENT_FAILED if isinstance(state, Exception): raise state return Response( @@ -86,9 +96,14 @@ def create_client(**_): self.enterContext(patch.object(resource, "AsyncClient", side_effect=create_client)) - def assert_deleted(self): + def assert_deleted(self, failed_disable=False, failed_create=False): self.assertEqual( - [request.method for request in self.requests], ["POST", "GET", "PATCH", "GET", "DELETE", "GET"] + [request.method for request in self.requests], + ["POST", "GET"] + + (["GET"] if failed_create else []) + + ["PATCH", "GET"] + + (["GET"] if failed_disable else []) + + ["DELETE", "GET"], ) self.assertEqual(self.requests[-2].url.path, f"/api{self.resource_path}") @@ -103,16 +118,27 @@ async def test_failed_bootstrap_cleans_up_after_failed_disable(self): ) with self.assertRaises(AssertionError): await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) - self.assert_deleted() + self.assert_deleted(failed_disable=True, failed_create=True) - async def test_polling_error_still_cleans_up(self): + async def test_failed_disable_does_not_delete_an_enabled_resource(self): + self.mock_api( + strings.RESOURCE_STATUS_DEPLOYMENT_FAILED, + disable_outcome=strings.RESOURCE_STATUS_UPDATING_FAILED, + resource_disabled=False, + ) + with self.assertRaises(AssertionError) as caught: + await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) + self.assertIn("not confirmed disabled", str(caught.exception.__notes__)) + self.assertNotIn("DELETE", [request.method for request in self.requests]) + + async def test_transient_polling_error_waits_for_terminal_state_before_cleanup(self): # Use a non-retried error to verify cleanup without delaying the test. failure = RuntimeError("poll failed") self.mock_api(failure) with self.assertRaises(RuntimeError) as caught: await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assertIs(caught.exception, failure) - self.assert_deleted() + self.assert_deleted(failed_create=True) async def test_cleanup_failure_preserves_bootstrap_error(self): failure = RuntimeError("bootstrap failed") @@ -121,7 +147,7 @@ async def test_cleanup_failure_preserves_bootstrap_error(self): await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assertIs(caught.exception, failure) self.assertIn("also failed", caught.exception.__notes__[0]) - self.assert_deleted() + self.assert_deleted(failed_create=True) async def test_cleanup_failure_fails_a_successful_bootstrap(self): self.mock_api(strings.RESOURCE_STATUS_DEPLOYED, delete_outcome=strings.RESOURCE_STATUS_DELETING_FAILED) @@ -140,7 +166,7 @@ async def test_rejected_disable_does_not_delete(self): with self.assertRaises(AssertionError) as caught: await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) self.assertIn("also failed", caught.exception.__notes__[0]) - self.assertEqual([request.method for request in self.requests], ["POST", "GET", "PATCH"]) + self.assertEqual([request.method for request in self.requests], ["POST", "GET", "GET", "PATCH"]) async def test_successful_bootstrap_still_requires_successful_disable(self): self.mock_api(strings.RESOURCE_STATUS_DEPLOYED, disable_outcome=strings.RESOURCE_STATUS_UPDATING_FAILED) diff --git a/e2e_tests/unit_tests/test_linux_vm_nexus.py b/e2e_tests/unit_tests/test_linux_vm_nexus.py index d500509534..303e33a9c3 100644 --- a/e2e_tests/unit_tests/test_linux_vm_nexus.py +++ b/e2e_tests/unit_tests/test_linux_vm_nexus.py @@ -1,15 +1,18 @@ """Test Nexus prerequisites, explicit consent and ownership of cleanup resources.""" +import asyncio import json import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import AsyncMock, patch from httpx import AsyncClient, MockTransport, Response -from jsonschema import validate +from jsonschema import ValidationError, validate -from e2e_tests import helpers -from e2e_tests import test_guacamole_linuxvm as linuxvm +from api_app.services.schema_service import enrich_shared_service_template +from e2e_tests import helpers, conftest as fixtures +from e2e_tests.resources import nexus from e2e_tests.resources import resource, strings @@ -25,9 +28,13 @@ def setUp(self): ), strings.CERTS_SHARED_SERVICE: json.loads((templates / "certs/template_schema.json").read_text()), } + self.schemas = { + name: enrich_shared_service_template(SimpleNamespace(model_dump=lambda schema=schema, **_: schema)) + for name, schema in self.schemas.items() + } self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) - self.enterContext(patch.object(linuxvm, "get_admin_token", new=AsyncMock(return_value="admin-token"))) - self.enterContext(patch.object(linuxvm.config, "TEST_ACCEPT_NEXUS_EULA", True)) + self.enterContext(patch.object(nexus, "get_admin_token", new=AsyncMock(return_value="admin-token"))) + self.enterContext(patch.object(nexus.config, "TEST_ACCEPT_NEXUS_EULA", True)) self.enterContext( patch.object( resource, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(self.handle)) @@ -38,6 +45,8 @@ def handle(self, request): self.requests.append(request) if request.method == "GET" and request.url.path == strings.API_SHARED_SERVICES: return Response(200, json={"sharedServices": self.services}) + if request.method == "GET" and request.url.path in ("/api/shared-services/nexus", "/api/shared-services/certs"): + return Response(200, json={"sharedService": {"isEnabled": False}}) response_status = 200 if request.method == "POST": payload = json.loads(request.content) @@ -45,6 +54,7 @@ def handle(self, request): kind = "nexus" if payload["templateName"] == strings.NEXUS_SHARED_SERVICE else "certs" if kind == "nexus": self.assertIs(payload["properties"]["accept_nexus_eula"], True) + self.assertEqual(payload["properties"]["vm_size"], "Standard_D2s_v3") phase, response_status, state = "install", 202, "deploying" elif request.method == "GET": kind, phase = request.url.path.rsplit("/", 1)[-1].split("-") @@ -96,22 +106,22 @@ def deleted(self): async def test_existing_nexus_is_reused_without_consent_or_mutation(self): self.services = [self.nexus()] - with patch.object(linuxvm.config, "TEST_ACCEPT_NEXUS_EULA", False): - async with linuxvm.linux_vm_nexus(True): + with patch.object(nexus.config, "TEST_ACCEPT_NEXUS_EULA", False): + async with nexus.nexus_prerequisites(True): pass self.assertEqual([request.method for request in self.requests], ["GET"]) async def test_missing_nexus_requires_explicit_consent_before_creation(self): - with patch.object(linuxvm.config, "TEST_ACCEPT_NEXUS_EULA", False): + with patch.object(nexus.config, "TEST_ACCEPT_NEXUS_EULA", False): with self.assertRaisesRegex(AssertionError, "explicitly accept"): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("Prerequisites should not be ready") self.assertEqual([request.method for request in self.requests], ["GET"]) async def test_ambiguous_existing_nexus_is_not_mutated(self): self.services = [self.nexus(), self.nexus()] with self.assertRaisesRegex(AssertionError, "at most one Nexus"): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("Ambiguous ownership must stop setup") self.assertEqual([request.method for request in self.requests], ["GET"]) @@ -121,12 +131,12 @@ async def test_old_or_failed_nexus_is_not_silently_reused_or_replaced(self): self.services = [self.nexus(**changes)] self.requests.clear() with self.assertRaises(AssertionError): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("Unsuitable Nexus should be rejected") self.assertEqual([request.method for request in self.requests], ["GET"]) async def test_created_resources_are_deleted_in_reverse_dependency_order(self): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.assertEqual(self.deleted(), []) posts = [json.loads(request.content)["templateName"] for request in self.requests if request.method == "POST"] self.assertEqual(posts, [strings.CERTS_SHARED_SERVICE, strings.NEXUS_SHARED_SERVICE]) @@ -134,25 +144,38 @@ async def test_created_resources_are_deleted_in_reverse_dependency_order(self): async def test_existing_certificate_service_is_preserved(self): self.services = [self.certs()] - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): pass posts = [json.loads(request.content) for request in self.requests if request.method == "POST"] self.assertEqual(len(posts), 1) self.assertEqual(posts[0]["properties"]["ssl_cert_name"], "existing-cert") self.assertEqual(self.deleted(), ["nexus"]) + async def test_both_prerequisites_include_inherited_required_properties(self): + async with nexus.nexus_prerequisites(True): + pass + posts = [json.loads(request.content) for request in self.requests if request.method == "POST"] + self.assertEqual(len(posts), 2) + for payload in posts: + with self.subTest(template=payload["templateName"]): + schema = self.schemas[payload["templateName"]] + validate(payload["properties"], schema) + del payload["properties"]["description"] + with self.assertRaisesRegex(ValidationError, "'description' is a required property"): + validate(payload["properties"], schema) + async def test_unrelated_certificate_service_is_not_replaced(self): # The API allows only one active service per template. self.services = [self.certs(properties={"domain_prefix": "other", "cert_name": "unrelated"})] with self.assertRaisesRegex(AssertionError, "belongs to another domain"): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("An unrelated certificate service must be preserved") self.assertEqual([request.method for request in self.requests], ["GET"]) async def test_failed_existing_certificate_service_requires_repair(self): self.services = [self.certs(deploymentStatus="deployment_failed")] with self.assertRaisesRegex(AssertionError, "Repair the existing Nexus certificate service"): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("A failed certificate service must not be reused") self.assertEqual([request.method for request in self.requests], ["GET"]) @@ -160,14 +183,14 @@ async def test_failed_nexus_deployment_cleans_up_both_owned_resources(self): self.outcomes[("nexus", "install")] = "deployment_failed" self.outcomes[("nexus", "disable")] = "updating_failed" with self.assertRaises(AssertionError): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("A failed Nexus install must not be used") self.assertEqual(self.deleted(), ["nexus", "certs"]) async def test_failed_certificate_deployment_does_not_create_nexus(self): self.outcomes[("certs", "install")] = "deployment_failed" with self.assertRaises(AssertionError): - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): self.fail("A failed certificate install must stop setup") posts = [json.loads(request.content)["templateName"] for request in self.requests if request.method == "POST"] self.assertEqual(posts, [strings.CERTS_SHARED_SERVICE]) @@ -177,11 +200,44 @@ async def test_vm_failure_preserved_while_shared_resources_are_cleaned(self): failure = RuntimeError("VM bootstrap failed") bootstrap_vm = AsyncMock(side_effect=failure) with self.assertRaisesRegex(RuntimeError, "VM bootstrap failed") as caught: - async with linuxvm.linux_vm_nexus(True): + async with nexus.nexus_prerequisites(True): await bootstrap_vm() self.assertIs(caught.exception, failure) self.assertEqual(self.deleted(), ["nexus", "certs"]) + async def test_prompt_nexus_delete_failure_retains_its_certificate(self): + owned = nexus.nexus_prerequisites(True) + await owned.__aenter__() + self.services = [self.nexus(deploymentStatus="deleting_failed")] + self.outcomes[("nexus", "delete")] = "deleting_failed" + with self.assertRaises(AssertionError) as caught: + await owned.__aexit__(None, None, None) + self.assertEqual(self.deleted(), ["nexus"]) + self.assertIn("Retaining certificate dependency", str(caught.exception.__notes__)) + + async def test_certificate_lookup_failure_retains_the_dependency(self): + owned = nexus.nexus_prerequisites(True) + await owned.__aenter__() + with patch.object(nexus, "get_resource", AsyncMock(side_effect=RuntimeError("lookup failed"))): + with self.assertRaisesRegex(RuntimeError, "lookup failed"): + await owned.__aexit__(None, None, None) + self.assertEqual(self.deleted(), ["nexus"]) + + async def test_nested_cleanup_stops_at_one_deadline_and_retains_certificate(self): + owned = nexus.nexus_prerequisites(True) + await owned.__aenter__() + self.outcomes[("nexus", "disable")] = "updating" + with patch.object(fixtures, "CLEANUP_TIMEOUT_SECONDS", 0.01): + with self.assertRaises(TimeoutError): + async with fixtures.resource_cleanup_timeout("Nexus prerequisites"): + await asyncio.wait_for(owned.__aexit__(None, None, None), 0.5) + mutations = [(request.method, request.url.path) for request in self.requests if request.method != "GET"] + self.assertEqual(mutations[-1], ("PATCH", "/api/shared-services/nexus")) + self.assertNotIn(("PATCH", "/api/shared-services/certs"), mutations) + count = len(self.requests) + await asyncio.sleep(0.01) + self.assertEqual(len(self.requests), count) + if __name__ == "__main__": unittest.main() diff --git a/e2e_tests/unit_tests/test_main_test_selection.py b/e2e_tests/unit_tests/test_main_test_selection.py new file mode 100644 index 0000000000..fa299c2ed0 --- /dev/null +++ b/e2e_tests/unit_tests/test_main_test_selection.py @@ -0,0 +1,54 @@ +"""Check the main workflow's selectors against real E2E collection.""" + +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class MainTestSelectionTests(unittest.TestCase): + def collect_groups(self, selector): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "groups" + env = dict( + os.environ, + PYTHONPATH=os.pathsep.join((str(ROOT), str(ROOT / "e2e_tests"))), + E2E_SELECTOR=selector, + GITHUB_OUTPUT=str(output), + ) + env.pop("PYTEST_ADDOPTS", None) + result = subprocess.run( + [sys.executable, "ci_groups.py", "--plan"], + cwd=ROOT / "e2e_tests", + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return {group["name"]: set(group["tests"]) for group in json.loads(output.read_text().split("groups=")[-1])} + + def test_no_consent_excludes_only_nexus_dependencies(self): + workflow = (ROOT / ".github/workflows/deploy_tre.yml").read_text() + expression = workflow.split("e2eTestsCustomSelector: >-", 1)[1].split("acceptNexusEula:", 1)[0] + selectors = [value for value in re.findall(r"'([^']*)'", expression) if "extended" in value] + self.assertEqual(len(selectors), 3) + push, consent, no_consent = [self.collect_groups(selector) for selector in selectors] + review_vm = "test_airlock.py::test_airlock_review_vm_flow" + self.assertEqual(set(no_consent), {"other"}) + self.assertEqual(set(consent), {"other", "nexus"}) + self.assertEqual(len(consent["nexus"]), 1) + self.assertEqual(consent["other"] - no_consent["other"], {review_vm}) + self.assertEqual(no_consent["other"] - consent["other"], set()) + self.assertIn("test_airlock.py::test_draft_container_is_deleted_after_submit", no_consent["other"]) + self.assertIn("test_airlock_consolidated.py::test_v2_export_uses_workspace_storage", no_consent["other"]) + self.assertLess(push["other"], no_consent["other"]) + self.assertFalse(any(node.startswith("test_airlock") for node in push["other"])) diff --git a/e2e_tests/unit_tests/test_review_vm_cleanup.py b/e2e_tests/unit_tests/test_review_vm_cleanup.py new file mode 100644 index 0000000000..84790c8ac0 --- /dev/null +++ b/e2e_tests/unit_tests/test_review_vm_cleanup.py @@ -0,0 +1,252 @@ +"""Exercise review VM ownership after creation and during asynchronous deletion.""" + +import asyncio +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import AsyncClient, MockTransport, Response +from e2e_tests import helpers, test_airlock as airlock +from e2e_tests.resources import resource + + +class ReviewVmCleanupTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.requests = [] + self.state = "deployed" + self.active_operation = False + self.path = "/workspaces/review/workspace-services/guac/user-resources/vm" + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + self.enterContext(patch.object(airlock, "post_resource", AsyncMock(return_value=(self.path, "vm")))) + self.enterContext( + patch.object(airlock, "delete_owned_resource_if_present", resource.delete_owned_resource_if_present) + ) + self.enterContext( + patch.object( + resource, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(self.handle)) + ) + ) + + async def handle(self, request): + self.requests.append((request.method, request.url.path)) + self.assertEqual(request.headers["Authorization"], "Bearer review-token") + if request.method == "GET" and request.url.path == f"/api{self.path}": + if self.state == "absent": + return Response(404) + return Response(200, json={"userResource": {"deploymentStatus": self.state}}) + if request.url.path.endswith("/operations"): + return Response(200, json={"operations": [{"status": "deleting" if self.active_operation else "deployed"}]}) + if request.method == "PATCH": + phase, code, state = "disable", 202, "updating" + elif request.method == "DELETE": + phase, code, state = "delete", 200, "deleting" + else: + phase, code = request.url.path.rsplit("/", 1)[-1], 200 + state = {"disable": "updated", "delete": "deleted"}[phase] + return Response( + code, + headers={"Location": f"/api/operations/{phase}"}, + json={ + "operation": { + "resourceId": "vm", + "status": state, + "message": "test", + "steps": [], + } + }, + ) + + def owner(self): + return airlock.managed_review_vm("/workspaces/research", "request", "research-token", "review-token", True) + + async def test_successful_creation_is_cleaned_after_approval_failure(self): + error = RuntimeError("approval failed") + approve = AsyncMock(side_effect=error) + with self.assertRaises(RuntimeError) as caught: + async with self.owner() as owned: + self.assertEqual(owned, self.path) + await approve() + self.assertIs(caught.exception, error) + self.assertIn(("DELETE", f"/api{self.path}"), self.requests) + + async def test_body_cancellation_waits_for_child_cleanup(self): + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.01): + async with self.owner(): + await asyncio.sleep(1) + self.assertEqual(self.requests[-1], ("GET", "/api/operations/delete")) + + async def test_normal_airlock_deletion_does_not_send_a_second_delete(self): + self.state = "absent" + async with self.owner(): + pass + self.assertEqual(self.requests, [("GET", f"/api{self.path}")]) + + async def test_active_approval_deletion_finishes_before_cleanup_returns(self): + self.state, self.active_operation = "updating", True + + async def finish_operation(seconds): + self.assertFalse(any(method in ("PATCH", "DELETE") for method, _ in self.requests)) + self.state = "absent" + + with patch.object(resource.asyncio, "sleep", finish_operation): + async with self.owner(): + pass + self.assertEqual([method for method, _ in self.requests], ["GET", "GET", "GET"]) + + async def test_cleanup_failure_preserves_the_original_body_failure(self): + original = RuntimeError("approval failed") + approve = AsyncMock(side_effect=original) + with patch.object(airlock, "delete_owned_resource_if_present", AsyncMock(side_effect=TimeoutError("stalled"))): + with self.assertRaises(RuntimeError) as caught: + async with self.owner(): + await approve() + self.assertIs(caught.exception, original) + self.assertIn(self.path, str(caught.exception.__notes__)) + + async def test_outer_timeout_during_recovery_preserves_body_failure(self): + started, finished = asyncio.Event(), asyncio.Event() + original = TimeoutError("automatic deletion stalled") + + async def cleanup(*args): + started.set() + await asyncio.sleep(0.03) + finished.set() + + @airlock.async_test_timeout(0.01) + async def run(): + async with self.owner(): + raise original + + with patch.object(airlock, "delete_owned_resource_if_present", cleanup): + with self.assertRaises(TimeoutError) as caught: + await run() + self.assertTrue(started.is_set()) + self.assertTrue(finished.is_set()) + self.assertIs(caught.exception, original) + + async def test_repeated_cancellation_during_successful_exit_waits_for_cleanup(self): + started, release, finished = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def cleanup(*args): + started.set() + await release.wait() + finished.set() + + async def run(): + async with self.owner(): + pass + + with patch.object(airlock, "delete_owned_resource_if_present", cleanup): + task = asyncio.create_task(run()) + try: + await asyncio.wait_for(started.wait(), 1) + for _ in range(2): + task.cancel() + await asyncio.sleep(0) + self.assertFalse(task.done()) + release.set() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, 1) + self.assertTrue(finished.is_set()) + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) + + async def test_cleanup_deadline_after_outer_timeout_preserves_body_failure(self): + original = RuntimeError("approval failed") + stopped = asyncio.Event() + + async def cleanup(*args): + try: + await asyncio.Event().wait() + finally: + stopped.set() + + @airlock.async_test_timeout(0.01) + async def run(): + async with self.owner(): + raise original + + with ( + patch.object(airlock, "delete_owned_resource_if_present", cleanup), + patch.object(airlock, "cleanup_deadline", side_effect=lambda _: resource.cleanup_deadline(0.03)), + ): + with self.assertRaises(RuntimeError) as caught: + await run() + self.assertIs(caught.exception, original) + self.assertTrue(stopped.is_set()) + self.assertIn("TimeoutError", str(original.__notes__)) + + async def test_cleanup_deadline_failure_is_reported_after_successful_body(self): + with ( + patch.object(airlock, "delete_owned_resource_if_present", AsyncMock(side_effect=TimeoutError("stalled"))), + self.assertRaisesRegex(TimeoutError, "stalled"), + ): + async with self.owner(): + pass + + async def test_cancelled_recovery_preserves_the_original_failure(self): + original = RuntimeError("approval failed") + with patch.object(airlock, "delete_owned_resource_if_present", AsyncMock(side_effect=asyncio.CancelledError)): + with self.assertRaises(RuntimeError) as caught: + async with self.owner(): + raise original + self.assertIs(caught.exception, original) + self.assertIn("CancelledError", str(original.__notes__)) + + +class ReviewVmDeletionTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + self.requests = [] + + async def run_states(self, states, **kwargs): + remaining = iter(states) + + def respond(request): + self.requests.append(request.method) + self.assertEqual(request.headers["Authorization"], "Bearer review-token") + state = next(remaining) + if isinstance(state, int): + return Response(state) + return Response(200, json={"userResource": {"deploymentStatus": state}}) + + with ( + patch.object(airlock, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(respond))), + patch.object(airlock.asyncio, "sleep", AsyncMock()), + ): + await airlock.wait_for_review_vm_deletion( + "/workspaces/review/user-resources/vm", "review-token", True, **kwargs + ) + + async def test_fast_deletion_does_not_require_observing_updating(self): + for states in ([404], ["awaiting_deletion", "deleting", 404], ["deleted"]): + with self.subTest(states=states): + self.requests.clear() + await self.run_states(states) + self.assertEqual(self.requests, ["GET"] * len(states)) + + async def test_waits_through_disable_and_delete_progress(self): + states = ["deployed", "awaiting_update", "updating", "updated", "awaiting_deletion", "deleting", 404] + await self.run_states(states) + self.assertEqual(len(self.requests), len(states)) + + async def test_failed_deletion_is_not_reported_as_success(self): + for state in ("updating_failed", "deleting_failed", "pipeline_failed"): + with self.subTest(state=state): + with self.assertRaisesRegex(AssertionError, state): + await self.run_states([state]) + + async def test_server_error_is_not_treated_as_deleted(self): + with self.assertRaises(AssertionError): + await self.run_states([500]) + + async def test_stalled_deletion_has_a_finite_deadline(self): + def respond(request): + return Response(200, json={"userResource": {"deploymentStatus": "deleting"}}) + + with patch.object( + airlock, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(respond)) + ): + with self.assertRaisesRegex(TimeoutError, "Last state: deleting"): + await airlock.wait_for_review_vm_deletion("/vm", "review-token", True, timeout_seconds=0.01) diff --git a/e2e_tests/unit_tests/test_review_workspace_ownership.py b/e2e_tests/unit_tests/test_review_workspace_ownership.py new file mode 100644 index 0000000000..fe14eb9573 --- /dev/null +++ b/e2e_tests/unit_tests/test_review_workspace_ownership.py @@ -0,0 +1,76 @@ +"""Check ownership when an Airlock review workspace is reused.""" + +import unittest +from unittest.mock import AsyncMock, patch + +from e2e_tests import conftest as fixtures + + +class ReviewWorkspaceOwnershipTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.object(fixtures.config, "TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_ID", "existing")) + self.enterContext(patch.object(fixtures.config, "TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_SERVICE_ID", "")) + self.enterContext( + patch.object( + fixtures, "create_or_get_test_workspace", AsyncMock(return_value=("/workspaces/existing", "existing")) + ) + ) + self.enterContext(patch.object(fixtures, "get_admin_token", AsyncMock(return_value="admin"))) + self.enterContext(patch.object(fixtures, "get_workspace_auth_details", AsyncMock(return_value=("owner", {})))) + self.create_service = self.enterContext( + patch.object( + fixtures, + "create_or_get_test_workpace_service", + AsyncMock(return_value=("/workspaces/existing/workspace-services/new", "new")), + ) + ) + self.delete_service = self.enterContext(patch.object(fixtures, "disable_and_delete_ws_resource", AsyncMock())) + self.delete_workspace = self.enterContext( + patch.object(fixtures, "disable_and_delete_tre_resource", AsyncMock()) + ) + + def fixture(self): + return fixtures.setup_test_airlock_import_review_workspace_and_guacamole_service.__wrapped__(None, True, None) + + async def finish(self, fixture): + with self.assertRaises(StopAsyncIteration): + await anext(fixture) + + async def test_created_child_is_removed_and_reused_parent_is_preserved(self): + fixture = self.fixture() + await anext(fixture) + await self.finish(fixture) + self.delete_service.assert_awaited_once_with("/workspaces/existing/workspace-services/new", "existing", True) + self.delete_workspace.assert_not_awaited() + + async def test_reused_child_is_preserved(self): + with patch.object(fixtures.config, "TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_SERVICE_ID", "existing-child"): + fixture = self.fixture() + await anext(fixture) + await self.finish(fixture) + self.delete_service.assert_not_awaited() + self.delete_workspace.assert_not_awaited() + + async def test_owned_parent_uses_cascading_cleanup(self): + with patch.object(fixtures.config, "TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_ID", ""): + fixture = self.fixture() + await anext(fixture) + await self.finish(fixture) + self.delete_service.assert_not_awaited() + self.delete_workspace.assert_awaited_once_with("/workspaces/existing", True) + + async def test_failed_child_cleanup_still_calls_parent_cleanup(self): + self.delete_service.side_effect = RuntimeError("child delete failed") + with patch.object(fixtures, "clean_up_test_workspace", AsyncMock()) as parent: + fixture = self.fixture() + await anext(fixture) + with self.assertRaisesRegex(RuntimeError, "child delete failed"): + await anext(fixture) + parent.assert_awaited_once() + + async def test_failed_child_setup_does_not_delete_reused_workspace(self): + self.create_service.side_effect = RuntimeError("service setup failed") + with self.assertRaisesRegex(RuntimeError, "service setup failed"): + await anext(self.fixture()) + self.delete_service.assert_not_awaited() + self.delete_workspace.assert_not_awaited() diff --git a/e2e_tests/unit_tests/test_shared_service_admin_vm.py b/e2e_tests/unit_tests/test_shared_service_admin_vm.py new file mode 100644 index 0000000000..808dbbd6f4 --- /dev/null +++ b/e2e_tests/unit_tests/test_shared_service_admin_vm.py @@ -0,0 +1,32 @@ +"""Prevent shared-service tests from reverting to the restricted admin VM size.""" + +import json +from pathlib import Path +import unittest +from unittest.mock import AsyncMock, patch + +from jsonschema import validate + +from e2e_tests import test_shared_services as shared + + +class SharedServiceAdminVmTests(unittest.IsolatedAsyncioTestCase): + async def test_admin_vm_payload_selects_available_schema_valid_size(self): + with ( + patch.object(shared, "disable_and_delete_shared_service_if_exists", new=AsyncMock()), + patch.object(shared, "get_admin_token", new=AsyncMock(return_value="token")), + patch.object( + shared, + "post_resource", + new=AsyncMock(return_value=("/shared-services/admin-vm", "admin-vm")), + ) as post, + patch.object(shared, "disable_and_delete_tre_resource", new=AsyncMock()), + ): + await shared.test_create_shared_service(shared.strings.ADMIN_VM_SHARED_SERVICE, True) + + post.assert_awaited_once() + payload = post.await_args.kwargs["payload"] + self.assertEqual(payload["templateName"], shared.strings.ADMIN_VM_SHARED_SERVICE) + self.assertEqual(payload["properties"].get("admin_jumpbox_vm_sku"), "Standard_D2s_v3") + schema_file = Path(__file__).resolve().parents[2] / "templates/shared_services/admin-vm/template_schema.json" + validate(payload["properties"], json.loads(schema_file.read_text())) diff --git a/e2e_tests/unit_tests/test_shared_service_nexus.py b/e2e_tests/unit_tests/test_shared_service_nexus.py new file mode 100644 index 0000000000..004440821f --- /dev/null +++ b/e2e_tests/unit_tests/test_shared_service_nexus.py @@ -0,0 +1,297 @@ +"""Check explicit Nexus consent before shared-service lifecycle mutations.""" + +import asyncio +import json +import os +import re +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import AsyncMock, patch + +import pytest +from jsonschema import validate + +from e2e_tests import test_shared_services as shared + + +class SharedServiceNexusConsentTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.date = self.enterContext(patch.object(shared, "date")) + self.date.today.return_value.weekday.return_value = 0 + self.enterContext(patch.object(shared.config, "TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS", False)) + self.cleanup_existing = self.enterContext( + patch.object(shared, "disable_and_delete_shared_service_if_exists", new=AsyncMock()) + ) + self.token = self.enterContext(patch.object(shared, "get_admin_token", new=AsyncMock(return_value="token"))) + self.find_service = self.enterContext( + patch.object(shared, "get_shared_service_by_name", AsyncMock(return_value=None)) + ) + self.post = self.enterContext( + patch.object( + shared, + "post_resource", + new=AsyncMock(side_effect=[("/shared-services/certs", "certs"), ("/shared-services/nexus", "nexus")]), + ) + ) + self.cleanup_created = self.enterContext( + patch.object(shared, "disable_and_delete_tre_resource", new=AsyncMock()) + ) + + async def test_missing_consent_fails_before_azure_access_even_on_weekends(self): + for day in (0, 5, 6): + with self.subTest(day=day), patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", False): + self.date.today.return_value.weekday.return_value = day + with self.assertRaisesRegex(pytest.fail.Exception, "acceptance is required.*coverage is unproven"): + await shared.test_create_certs_nexus_shared_service(True) + self.cleanup_existing.assert_not_awaited() + self.token.assert_not_awaited() + self.post.assert_not_awaited() + self.cleanup_created.assert_not_awaited() + + async def test_previous_resources_share_one_recovery_deadline_before_new_creation(self): + async def recover(name, verify): + if name == shared.strings.NEXUS_SHARED_SERVICE: + await asyncio.sleep(0.01) + else: + await asyncio.sleep(1) + + self.cleanup_existing.side_effect = recover + with ( + patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True), + patch.object(shared, "RECOVERY_TIMEOUT_SECONDS", 0.03), + ): + with self.assertRaises(TimeoutError): + await shared.test_create_certs_nexus_shared_service(True) + self.assertEqual(self.cleanup_existing.await_count, 2) + self.post.assert_not_awaited() + self.cleanup_created.assert_not_awaited() + + async def test_explicit_consent_reaches_schema_valid_nexus_payload(self): + with patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True): + await shared.test_create_certs_nexus_shared_service(True) + payloads = [call.kwargs["payload"] for call in self.post.await_args_list] + self.assertEqual( + [payload["templateName"] for payload in payloads], + [ + shared.strings.CERTS_SHARED_SERVICE, + shared.strings.NEXUS_SHARED_SERVICE, + ], + ) + schema_file = ( + Path(__file__).resolve().parents[2] / "templates/shared_services/sonatype-nexus-vm/template_schema.json" + ) + validate(payloads[1]["properties"], json.loads(schema_file.read_text())) + self.assertIs(payloads[1]["properties"]["accept_nexus_eula"], True) + self.assertEqual(payloads[1]["properties"].get("vm_size"), "Standard_D2s_v3") + self.assertEqual( + [call.args[0] for call in self.cleanup_created.await_args_list], + ["/shared-services/nexus", "/shared-services/certs"], + ) + + async def test_weekend_skip_with_consent_still_leaves_resources_unchanged(self): + self.date.today.return_value.weekday.return_value = 5 + with patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True): + with self.assertRaisesRegex(pytest.skip.Exception, "coverage is unproven"): + await shared.test_create_certs_nexus_shared_service(True) + self.cleanup_existing.assert_not_awaited() + self.token.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_explicit_weekend_override_runs_certificate_lifecycle(self): + self.date.today.return_value.weekday.return_value = 5 + with ( + patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True), + patch.object(shared.config, "TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS", True), + ): + await shared.test_create_certs_nexus_shared_service(True) + self.assertEqual(self.post.await_count, 2) + self.assertEqual(self.cleanup_created.await_count, 2) + + async def test_provisioning_timeout_leaves_time_for_certificate_cleanup(self): + async def deploy(*, payload, **kwargs): + if payload["templateName"] == shared.strings.CERTS_SHARED_SERVICE: + return "/shared-services/certs", "certs" + await asyncio.sleep(60) + return "/shared-services/nexus", "nexus" + + async def cleanup(*args): + await asyncio.sleep(0.03) + + self.post.side_effect = deploy + self.cleanup_created.side_effect = cleanup + with ( + patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True), + patch.object(shared, "PROVISIONING_TIMEOUT_SECONDS", 0.01), + ): + with self.assertRaises(TimeoutError): + await shared.test_create_certs_nexus_shared_service(True) + self.cleanup_created.assert_awaited_once_with("/shared-services/certs", True) + + async def test_successful_provisioning_cleanup_outlives_provisioning_deadline(self): + async def cleanup(*args): + await asyncio.sleep(0.02) + + self.cleanup_created.side_effect = cleanup + with ( + patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True), + patch.object(shared, "PROVISIONING_TIMEOUT_SECONDS", 0.01), + ): + await shared.test_create_certs_nexus_shared_service(True) + self.assertEqual( + [call.args[0] for call in self.cleanup_created.await_args_list], + ["/shared-services/nexus", "/shared-services/certs"], + ) + + async def test_cleanup_timeout_is_reported_and_remaining_cleanup_is_attempted(self): + async def cleanup(path, verify): + if path.endswith("nexus"): + await asyncio.sleep(60) + + self.cleanup_created.side_effect = cleanup + with ( + patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True), + patch.object(shared, "CLEANUP_TIMEOUT_SECONDS", 0.01), + ): + with self.assertRaises(TimeoutError): + await shared.test_create_certs_nexus_shared_service(True) + self.assertEqual(self.cleanup_created.await_count, 2) + + async def test_cleanup_failure_does_not_replace_provisioning_error(self): + failure = RuntimeError("Nexus deployment failed") + self.post.side_effect = [("/shared-services/certs", "certs"), failure] + self.cleanup_created.side_effect = RuntimeError("Certificate cleanup failed") + with patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True): + with self.assertRaises(RuntimeError) as caught: + await shared.test_create_certs_nexus_shared_service(True) + self.assertIs(caught.exception, failure) + self.assertIn("/shared-services/certs", caught.exception.__notes__[0]) + + async def test_certificate_is_retained_when_nexus_cleanup_failed(self): + failure = RuntimeError("Nexus cleanup failed") + self.cleanup_created.side_effect = failure + self.find_service.return_value = {"id": "nexus"} + with patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", True): + with self.assertRaises(RuntimeError) as caught: + await shared.test_create_certs_nexus_shared_service(True) + self.assertIs(caught.exception, failure) + self.cleanup_created.assert_awaited_once_with("/shared-services/nexus", True) + self.assertIn("Retaining certificate dependency", caught.exception.__notes__[0]) + + +class ExistingSharedServiceRecoveryTests(unittest.IsolatedAsyncioTestCase): + async def test_unrelated_service_is_preserved(self): + service = {"id": "existing", "properties": {"display_name": "Retain this service"}} + with ( + patch.object(shared, "get_admin_token", AsyncMock(return_value="token")), + patch.object(shared, "get_shared_service_by_name", AsyncMock(return_value=service)), + patch.object(shared, "disable_and_delete_resource", AsyncMock()) as cleanup, + ): + with self.assertRaisesRegex(AssertionError, "not created by this test"): + await shared.disable_and_delete_shared_service_if_exists(shared.strings.CERTS_SHARED_SERVICE, True) + cleanup.assert_not_awaited() + + async def test_old_test_resource_waits_for_terminal_operation_before_recovery(self): + name = shared.strings.CERTS_SHARED_SERVICE + service = { + "id": "certs", + "properties": { + "display_name": f"Shared service {name}", + "description": f"{name} deployed via e2e tests", + }, + } + events = [] + + async def operations(*args): + events.append("poll") + return {"operations": [{"status": "pipeline_running" if len(events) == 1 else "updated"}]} + + async def cleanup(*args, **kwargs): + events.append("delete") + + with ( + patch.object(shared, "get_admin_token", AsyncMock(return_value="token")), + patch.object(shared, "get_shared_service_by_name", AsyncMock(return_value=service)), + patch.object(shared, "get_resource", operations), + patch.object(shared.asyncio, "sleep", AsyncMock()), + patch.object(shared, "disable_and_delete_resource", cleanup), + ): + await shared.disable_and_delete_shared_service_if_exists(name, True) + self.assertEqual(events, ["poll", "poll", "delete"]) + + +class NexusConsentConfigurationTests(unittest.TestCase): + def test_absent_false_and_explicit_true_configuration(self): + root = Path(__file__).resolve().parents[2] + for value, expected in ((None, "False"), ("false", "False"), ("true", "True")): + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + env = dict(os.environ, PYTHONPATH=str(root)) + env.pop("TEST_ACCEPT_NEXUS_EULA", None) + if value is not None: + env["TEST_ACCEPT_NEXUS_EULA"] = value + result = subprocess.run( + [ + sys.executable, + "-c", + "from e2e_tests.config import TEST_ACCEPT_NEXUS_EULA; print(TEST_ACCEPT_NEXUS_EULA)", + ], + env=env, + cwd=directory, + capture_output=True, + text=True, + check=True, + timeout=10, + ) + self.assertEqual(result.stdout.strip(), expected) + + +class SharedServiceWorkflowBudgetTests(unittest.TestCase): + def test_outer_watchdogs_include_failed_create_and_final_cleanup_with_job_margin(self): + single = next(mark.args[0] for mark in shared.test_create_shared_service.pytestmark if mark.name == "timeout") + nexus = next( + mark.args[0] for mark in shared.test_create_certs_nexus_shared_service.pytestmark if mark.name == "timeout" + ) + setup = shared.RECOVERY_TIMEOUT_SECONDS + shared.PROVISIONING_TIMEOUT_SECONDS + self.assertGreater( + single, setup + max(shared.CLEANUP_TIMEOUT_SECONDS, shared.FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS) + ) + self.assertGreater(nexus, setup + 2 * shared.CLEANUP_TIMEOUT_SECONDS) + self.assertGreater(nexus, setup + shared.FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS + shared.CLEANUP_TIMEOUT_SECONDS) + workflow = (Path(__file__).resolve().parents[2] / ".github/workflows/deploy_tre_reusable.yml").read_text() + job_minutes = int(re.search(r"(?s) e2e_tests_custom:\n.*? timeout-minutes: (\d+)", workflow).group(1)) + self.assertGreaterEqual(job_minutes * 60 - nexus, 30 * 60) + + def test_main_selection_without_consent_excludes_only_nexus_from_shared_services(self): + root = Path(__file__).resolve().parents[2] + environment = dict(os.environ, PYTHONPATH=os.pathsep.join((str(root), str(root / "e2e_tests")))) + environment.pop("PYTEST_ADDOPTS", None) + for selector, expected_count in (("shared_services", 5), ("shared_services and not nexus", 4)): + with self.subTest(selector=selector): + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "-o", + "addopts=", + "-m", + selector, + "test_shared_services.py", + ], + cwd=root / "e2e_tests", + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + selected = [line for line in result.stdout.splitlines() if line.startswith("test_shared_services.py::")] + self.assertEqual(len(selected), expected_count, result.stdout) + self.assertEqual( + any("test_create_certs_nexus_shared_service" in line for line in selected), expected_count == 5 + ) diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml index 742091dd3f..5aae3ab735 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-export-reviewvm -version: 2.0.7 +version: 2.0.9 description: "An Azure TRE User Resource Template for reviewing Airlock export requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -9,10 +9,10 @@ registry: azuretre custom: # For information on vm_sizes and image_options, see README.me in the guacamole/user-resources folder vm_sizes: - "2 CPU | 8GB RAM": Standard_D2s_v7 - "4 CPU | 16GB RAM": Standard_D4s_v7 - "8 CPU | 32GB RAM": Standard_D8s_v7 - "16 CPU | 64GB RAM": Standard_D16s_v7 + "2 CPU | 8GB RAM": Standard_D2s_v6 + "4 CPU | 16GB RAM": Standard_D4s_v6 + "8 CPU | 32GB RAM": Standard_D8s_v6 + "16 CPU | 64GB RAM": Standard_D16s_v6 image_options: "Windows Server 2025": source_image_reference: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml index 8bacaa7ae9..36c78cdc4f 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-import-reviewvm -version: 2.0.4 +version: 2.0.6 description: "An Azure TRE User Resource Template for reviewing Airlock import requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -9,10 +9,10 @@ registry: azuretre custom: # For information on vm_sizes and image_options, see README.me in the guacamole/user-resources folder vm_sizes: - "2 CPU | 8GB RAM": Standard_D2s_v7 - "4 CPU | 16GB RAM": Standard_D4s_v7 - "8 CPU | 32GB RAM": Standard_D8s_v7 - "16 CPU | 64GB RAM": Standard_D16s_v7 + "2 CPU | 8GB RAM": Standard_D2s_v6 + "4 CPU | 16GB RAM": Standard_D4s_v6 + "8 CPU | 32GB RAM": Standard_D8s_v6 + "16 CPU | 64GB RAM": Standard_D16s_v6 image_options: "Windows Server 2025": source_image_reference: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf index 074ac3b848..389e7027df 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf @@ -28,6 +28,7 @@ module "windows_vm" { nexus_proxy_url = local.nexus_proxy_url install_azure_cli = true + require_azure_cli = true install_vscode = true install_storage_explorer = true install_git = false diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index dfc9e326ff..7bf342ff5c 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-linuxvm -version: 1.4.8 +version: 1.4.9 description: "An Azure TRE User Resource Template for Guacamole (Linux)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -9,10 +9,10 @@ registry: azuretre custom: # For information on vm_sizes and image_options, see README.me in the guacamole/user-resources folder vm_sizes: - "2 CPU | 8GB RAM": Standard_D2s_v7 - "4 CPU | 16GB RAM": Standard_D4s_v7 - "8 CPU | 32GB RAM": Standard_D8s_v7 - "16 CPU | 64GB RAM": Standard_D16s_v7 + "2 CPU | 8GB RAM": Standard_D2s_v6 + "4 CPU | 16GB RAM": Standard_D4s_v6 + "8 CPU | 32GB RAM": Standard_D8s_v6 + "16 CPU | 64GB RAM": Standard_D16s_v6 image_options: "Ubuntu 22.04 LTS": source_image_reference: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index e7b2686f5c..cb7aef8d6f 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-windowsvm -version: 3.0.4 +version: 3.0.5 description: "An Azure TRE User Resource Template for Guacamole (Windows 11 or Windows Server 2025)" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf index 4e653f2f66..2298b896b7 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf @@ -36,6 +36,7 @@ locals { nexus_proxy_url = var.nexus_proxy_url SharedStorageAccess = var.shared_storage_access ? 1 : 0 InstallAzureCli = var.install_azure_cli ? 1 : 0 + RequireAzureCli = var.require_azure_cli ? 1 : 0 InstallVsCode = var.install_vscode ? 1 : 0 InstallStorageExplorer = var.install_storage_explorer ? 1 : 0 InstallGit = var.install_git ? 1 : 0 diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf index 581ebb3d57..8dfc640804 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf @@ -47,6 +47,11 @@ variable "install_azure_cli" { type = bool default = true } +variable "require_azure_cli" { + description = "Fail bootstrap when Nexus or Azure CLI is unavailable, as required by Airlock review VMs" + type = bool + default = false +} variable "install_vscode" { type = bool default = true diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 index 305dc1bdbe..725f968d96 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 @@ -28,7 +28,7 @@ trusted-host = $NexusHost $Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False [System.IO.File]::WriteAllLines($PipConfigFilePath, $ConfigBody, $Utf8NoBomEncoding) -# Docker proxy config (best-effort — only applies if Docker is pre-installed) +# Docker proxy config (best-effort; only applies if Docker is pre-installed) try { $dockerConfigDir = "$env:ProgramData\docker\config" if (!(Test-Path $dockerConfigDir)) { @@ -55,6 +55,7 @@ $RStudioVersion = "2026.07.1-147" $GitVersion = "2.55.0" $MiniforgePath = "C:\Miniforge3" $InstallAzureCli = (${InstallAzureCli} -eq 1) +$RequireAzureCli = (${RequireAzureCli} -eq 1) $InstallVsCode = (${InstallVsCode} -eq 1) $InstallStorageExplorer = (${InstallStorageExplorer} -eq 1) $InstallGit = (${InstallGit} -eq 1) @@ -103,10 +104,11 @@ function Install-TreTool { [Parameter(Mandatory = $true)][string]$OutFile, [Parameter(Mandatory = $true)][string]$FilePath, [Parameter(Mandatory = $true)][string[]]$ArgumentList, - [int]$MaxAttempts = 5 + [int]$MaxAttempts = 5, + [switch]$Required ) - # Skip installs on failure rather than aborting the whole VM configuration. + # Optional tools may fail without aborting the VM configuration. try { Write-Host "Downloading $Name from $Url" for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) { @@ -122,11 +124,13 @@ function Install-TreTool { } Write-Host "Installing $Name" $process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -Wait -PassThru -NoNewWindow - if ($process.ExitCode -ne 0) { + if ($process.ExitCode -notin @(0, 3010)) { + if ($Required) { throw "$Name installer exited with code $($process.ExitCode)" } Write-Host "WARNING: $Name installer exited with code $($process.ExitCode)" } } catch { + if ($Required) { throw "Required tool $Name could not be installed: $($_.Exception.Message)" } Write-Host "WARNING: Failed to install $Name - $($_.Exception.Message)" } } @@ -205,7 +209,10 @@ function New-DesktopShortcut { # Make sure the proxy is up before pulling any of the installers below. if ($NexusActionsEnabled) { - Wait-ForNexus | Out-Null + $NexusReady = Wait-ForNexus + if ($RequireAzureCli -and -not $NexusReady) { + throw "Airlock review requires a reachable Nexus proxy. Check Nexus deployment, DNS and HTTPS connectivity." + } } else { Write-Host "No Nexus-backed install/config actions enabled - skipping the wait for the Nexus proxy." @@ -221,14 +228,35 @@ if ($ConfigureConda) Configure-RProxy $AzureCliMsi = Join-Path $ToolsDir "azure-cli.msi" +if ($RequireAzureCli -and -not $InstallAzureCli) { + throw "Airlock review requires Azure CLI installation to be enabled." +} if ($InstallAzureCli) { + if ($RequireAzureCli) { + # Probe the required package inside the workspace network before installing tools. + try { + Invoke-WebRequest -Uri "${nexus_proxy_url}/repository/azure-cli/azure-cli-$AzureCliVersion-x64.msi" ` + -Method Head -UseBasicParsing -TimeoutSec 30 | Out-Null + } + catch { + throw "Required Azure CLI package is unavailable through Nexus: $($_.Exception.Message)" + } + } Install-TreTool -Name "Azure CLI" ` -Url "${nexus_proxy_url}/repository/azure-cli/azure-cli-$AzureCliVersion-x64.msi" ` -OutFile $AzureCliMsi ` -FilePath "msiexec.exe" ` - -ArgumentList @("/i", "`"$AzureCliMsi`"", "/qn", "/norestart") + -ArgumentList @("/i", "`"$AzureCliMsi`"", "/qn", "/norestart") ` + -Required:$RequireAzureCli $env:Path = "$Env:ProgramFiles\Microsoft SDKs\Azure\CLI2\wbin;$Env:ProgramFiles (x86)\Microsoft SDKs\Azure\CLI2\wbin;$env:Path" + if ($RequireAzureCli) { + if (-not (Get-Command az -ErrorAction SilentlyContinue)) { + throw "Required Azure CLI executable was not found after installation." + } + & az version --output json | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Required Azure CLI failed its version check." } + } } $VsCodeSetup = Join-Path $ToolsDir "vscode-setup.exe" From b3aac8b9a1a2b2ddeb815468abd38568e2973992 Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:42:10 +0100 Subject: [PATCH 06/17] Select bundle tests and require successful smoke tests (#5119) (cherry picked from commit 2257a76ef07441c757879f145f55f04546e50e2f) --- .../scripts/ci_environment_workflow.test.js | 96 ++++ .github/scripts/package-lock.json | 4 +- .github/scripts/package.json | 2 +- .github/workflows/deploy_tre_branch.yml | 21 + .github/workflows/deploy_tre_reusable.yml | 87 +++- .github/workflows/e2e_helper_tests.yml | 1 + CHANGELOG.md | 1 + Makefile | 14 +- docs/tre-developers/end-to-end-tests.md | 84 ++++ docs/tre-developers/github-pr-bot-commands.md | 33 +- e2e_tests/.gitignore | 5 + e2e_tests/bundle_coverage.json | 462 ++++++++++++++++++ e2e_tests/bundle_evidence.py | 117 +++++ e2e_tests/pytest.ini | 1 + e2e_tests/resources/nexus.py | 10 + e2e_tests/resources/resource.py | 4 + e2e_tests/run_bundle.py | 229 +++++++++ e2e_tests/test_shared_services.py | 47 ++ e2e_tests/test_workspace_services.py | 1 - e2e_tests/unit_tests/README.md | 4 + e2e_tests/unit_tests/test_bundle_selection.py | 417 ++++++++++++++++ .../unit_tests/test_certificate_selection.py | 163 ++++++ .../test_reused_prerequisite_evidence.py | 76 +++ 23 files changed, 1866 insertions(+), 13 deletions(-) create mode 100644 e2e_tests/.gitignore create mode 100644 e2e_tests/bundle_coverage.json create mode 100644 e2e_tests/bundle_evidence.py create mode 100644 e2e_tests/run_bundle.py create mode 100644 e2e_tests/unit_tests/test_bundle_selection.py create mode 100644 e2e_tests/unit_tests/test_certificate_selection.py create mode 100644 e2e_tests/unit_tests/test_reused_prerequisite_evidence.py diff --git a/.github/scripts/ci_environment_workflow.test.js b/.github/scripts/ci_environment_workflow.test.js index d1554df810..9d2400963d 100644 --- a/.github/scripts/ci_environment_workflow.test.js +++ b/.github/scripts/ci_environment_workflow.test.js @@ -292,3 +292,99 @@ describe('regional CI workflow routing', () => { } }); }); + +describe('bundle and marker workflow integration', () => { + const reusable = workflow('deploy_tre_reusable.yml'); + const evaluate = (expression, inputs, results = ['success'], needs = { + e2e_tests_smoke: { result: 'success' }, e2e_tests_plan: { result: 'success' }, + verify_deployment_lock: { result: 'success' } + }) => vm.runInNewContext( + expression.trim().slice(3, -2).replaceAll('needs.*.result', 'results'), + { inputs, results, needs, always: () => true, contains: (values, value) => values.includes(value) }, + { timeout: 1000 } + ); + + test.each([ + ['tre-workspace-service-azuresql', '', false, true], + ['', 'shared_services', true, false], + ['', '', false, false] + ])('bundle %s and marker %s select only the requested path', (bundle, marker, planned, selected) => { + const inputs = { e2eBundle: bundle, e2eTestsCustomSelector: marker, skipDeployment: false }; + expect(evaluate(reusable.jobs.e2e_tests_plan.if, inputs)).toBe(planned); + expect(evaluate(reusable.jobs.e2e_tests_custom.if, inputs)).toBe(planned); + expect(evaluate(reusable.jobs.e2e_tests_bundle.if, inputs)).toBe(selected); + }); + + test('custom groups wait for smoke and their test plan', () => { + expect(reusable.jobs.e2e_tests_custom.needs).toEqual(expect.arrayContaining([ + 'e2e_tests_smoke', 'e2e_tests_plan' + ])); + }); + + describe.each([false, true])('custom groups with skipDeployment=%s', skipDeployment => { + const deploymentJobs = ['deploy_shared_services', 'register_bundles', 'register_user_resource_bundles']; + const canRun = overrides => { + const needs = Object.fromEntries(deploymentJobs.map(name => [name, { + result: skipDeployment ? 'skipped' : 'success' + }])); + needs.verify_deployment_lock = { result: 'success' }; + needs.e2e_tests_smoke = { result: 'success' }; + needs.e2e_tests_plan = { result: 'success' }; + for (const [name, result] of Object.entries(overrides)) needs[name] = { result }; + const inputs = { e2eBundle: '', e2eTestsCustomSelector: 'extended', skipDeployment }; + return evaluate(reusable.jobs.e2e_tests_custom.if, inputs, + Object.values(needs).map(job => job.result), needs); + }; + + test.each([ + ['success', true], ['failure', false], ['cancelled', false], ['skipped', false] + ])('smoke result %s permits custom execution: %s', (result, expected) => { + expect(canRun({ e2e_tests_smoke: result })).toBe(expected); + }); + + test.each(['failure', 'cancelled', 'skipped'])('test plan result %s blocks custom execution', result => { + expect(canRun({ e2e_tests_plan: result })).toBe(false); + }); + + test.each(deploymentJobs.flatMap(name => ['failure', 'cancelled', 'skipped'].map(result => [name, result])))( + '%s result %s only permits intentional deployment skips', (name, result) => { + expect(canRun({ [name]: result })).toBe(skipDeployment && result === 'skipped'); + }); + }); + + describe.each([false, true])('bundle execution with skipDeployment=%s', skipDeployment => { + test.each(['success', 'failure', 'cancelled', 'skipped'])('requires successful smoke tests: %s', result => { + const needs = Object.fromEntries(['deploy_shared_services', 'register_bundles', 'register_user_resource_bundles'] + .map(name => [name, { result: skipDeployment ? 'skipped' : 'success' }])); + needs.e2e_tests_smoke = { result }; + const inputs = { e2eBundle: 'tre-workspace-service-azuresql', e2eTestsCustomSelector: '', skipDeployment }; + expect(evaluate(reusable.jobs.e2e_tests_bundle.if, inputs, + Object.values(needs).map(job => job.result), needs)).toBe(result === 'success'); + }); + }); + + test('bundle execution has one job and no dependency on marker planning', () => { + const bundle = reusable.jobs.e2e_tests_bundle; + expect(bundle.strategy).toBeUndefined(); + expect(bundle.needs).toEqual(['deploy_shared_services', 'register_bundles', 'register_user_resource_bundles', 'e2e_tests_smoke']); + const prepare = bundle.steps.find(step => step.name === 'Prepare bundle validation'); + expect(prepare.env.TEST_BUNDLE).toBe('${{ inputs.e2eBundle }}'); + expect(prepare.env.TEST_BUNDLE_MARKER).toBe('${{ inputs.e2eTestsCustomSelector }}'); + expect(prepare.run).toBe('python3 e2e_tests/run_bundle.py --prepare'); + const run = bundle.steps.find(step => step.name === 'Run E2E Tests'); + expect(run.with.COMMAND).toContain('TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle'); + expect(run.with.COMMAND).not.toContain('inputs.e2eBundle'); + expect(bundle.steps.find(step => step.name === 'Upload Test Results').with.path).toContain('bundle-validation.json'); + expect(reusable.jobs.summary.needs).toEqual(['e2e_tests_smoke', 'e2e_tests_plan', 'e2e_tests_custom', 'e2e_tests_bundle']); + }); + + test('bundle execution honours deployment failures and intentional reuse', () => { + const inputs = { e2eBundle: 'tre-workspace-service-azuresql', skipDeployment: false }; + for (const result of ['failure', 'cancelled', 'skipped']) { + expect(evaluate(reusable.jobs.e2e_tests_bundle.if, inputs, [result])).toBe(false); + } + inputs.skipDeployment = true; + expect(evaluate(reusable.jobs.e2e_tests_bundle.if, inputs, ['skipped'])).toBe(true); + expect(evaluate(reusable.jobs.e2e_tests_bundle.if, inputs, ['failure'])).toBe(false); + }); +}); diff --git a/.github/scripts/package-lock.json b/.github/scripts/package-lock.json index f176019f64..d34eeb9acb 100644 --- a/.github/scripts/package-lock.json +++ b/.github/scripts/package-lock.json @@ -1,12 +1,12 @@ { "name": "build-scripts", - "version": "1.0.5", + "version": "1.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "build-scripts", - "version": "1.0.5", + "version": "1.1.3", "license": "MIT", "devDependencies": { "jest": "^30.2.0", diff --git a/.github/scripts/package.json b/.github/scripts/package.json index 8ce0ae331b..06cf6690be 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -1,6 +1,6 @@ { "name": "build-scripts", - "version": "1.0.5", + "version": "1.1.3", "description": "Scripts used in the GitHub workflows", "main": "build.js", "license": "MIT", diff --git a/.github/workflows/deploy_tre_branch.yml b/.github/workflows/deploy_tre_branch.yml index 31bf151d26..da4bb24a77 100644 --- a/.github/workflows/deploy_tre_branch.yml +++ b/.github/workflows/deploy_tre_branch.yml @@ -14,6 +14,11 @@ on: # yamllint disable-line rule:truthy type: string default: "" required: false + e2eBundle: + description: Exact Porter bundle name from the coverage map; leave the marker selector empty + type: string + default: "" + required: false environment: description: The environment to run this workflow in type: environment @@ -67,6 +72,21 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false + - name: Validate bundle selection + if: inputs.e2eBundle != '' + env: + TEST_BUNDLE: ${{ inputs.e2eBundle }} + TEST_BUNDLE_MARKER: ${{ inputs.e2eTestsCustomSelector }} + E2E_TESTS_NUMBER_PROCESSES: ${{ inputs.e2eProcesses }} + run: python3 e2e_tests/run_bundle.py --validate-only + + - name: Upload bundle selection preflight + if: always() && inputs.e2eBundle != '' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: Bundle Selection Preflight + path: ./e2e_tests/bundle-validation.json + - id: run-id name: Get run id env: @@ -104,6 +124,7 @@ jobs: ciEnvironmentId: ${{ needs.prepare-not-main.outputs.refid }} prHeadSha: ${{ github.sha }} e2eTestsCustomSelector: ${{ github.event.inputs.e2eTestsCustomSelector }} + e2eBundle: ${{ inputs.e2eBundle }} acceptNexusEula: ${{ inputs.acceptNexusEula }} runCertificateTestsOnWeekends: ${{ inputs.runCertificateTestsOnWeekends }} skipDeployment: ${{ inputs.skipDeployment }} diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 8cb81ea4ef..3ea66c973f 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -33,6 +33,11 @@ on: # yamllint disable-line rule:truthy type: string default: "" required: false + e2eBundle: + description: Exact Porter bundle name for isolated E2E case selection + type: string + default: "" + required: false acceptNexusEula: description: Explicit consent to the Nexus CE EULA when E2E tests create Nexus type: boolean @@ -948,7 +953,7 @@ jobs: name: Select E2E test groups # Same as smoke tests: skipDeployment intentionally bypasses deploy jobs but still runs E2E tests. if: >- - ${{ always() && inputs.e2eTestsCustomSelector != '' && + ${{ always() && inputs.e2eBundle == '' && inputs.e2eTestsCustomSelector != '' && (inputs.skipDeployment || !contains(needs.*.result, 'skipped')) && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} @@ -979,17 +984,18 @@ jobs: e2e_tests_custom: name: "Run E2E Tests (${{ matrix.group.name }})" - # Same as smoke tests: skipDeployment intentionally bypasses deploy jobs but still runs E2E tests. + # Deployment reuse can skip deployment jobs, but smoke and planning must succeed. if: >- - ${{ always() && inputs.e2eTestsCustomSelector != '' && + ${{ always() && inputs.e2eBundle == '' && inputs.e2eTestsCustomSelector != '' && needs.verify_deployment_lock.result == 'success' && + needs.e2e_tests_smoke.result == 'success' && needs.e2e_tests_plan.result == 'success' && (inputs.skipDeployment || !contains(needs.*.result, 'skipped')) && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} runs-on: ubuntu-latest environment: ${{ inputs.environmentName }} needs: - [verify_deployment_lock, deploy_shared_services, register_bundles, register_user_resource_bundles, e2e_tests_plan] + [verify_deployment_lock, deploy_shared_services, register_bundles, register_user_resource_bundles, e2e_tests_plan, e2e_tests_smoke] # Each serial group receives its own budget. Earlier cases cannot consume # the Nexus provisioning and cleanup window. timeout-minutes: 300 @@ -1042,9 +1048,80 @@ jobs: name: E2E Test Results (${{ matrix.group.name }}) path: "./e2e_tests/pytest_e2e_*.xml" + e2e_tests_bundle: + name: "Run E2E Tests (Bundle)" + # Same as smoke tests: skipDeployment intentionally bypasses deploy jobs but still runs E2E tests. + if: >- + ${{ always() && inputs.e2eBundle != '' && + needs.e2e_tests_smoke.result == 'success' && + (inputs.skipDeployment || !contains(needs.*.result, 'skipped')) && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') }} + runs-on: ubuntu-latest + environment: ${{ inputs.environmentName }} + # Smoke tests also write to TRE. Finish them before any selected bundle lifecycle. + needs: + [deploy_shared_services, register_bundles, register_user_resource_bundles, e2e_tests_smoke] + timeout-minutes: 300 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + # if the following values are missing (i.e. not triggered via comment workflow) + # then the default checkout will apply + ref: ${{ inputs.prRef }} + + - name: Prepare bundle validation + if: inputs.e2eBundle != '' + env: + TEST_BUNDLE: ${{ inputs.e2eBundle }} + TEST_BUNDLE_MARKER: ${{ inputs.e2eTestsCustomSelector }} + E2E_TESTS_NUMBER_PROCESSES: ${{ inputs.E2E_TESTS_NUMBER_PROCESSES }} + TEST_ACCEPT_NEXUS_EULA: ${{ inputs.acceptNexusEula }} + TEST_BUNDLE_CI_REF: ${{ inputs.ciGitRef }} + TRE_ID: ${{ secrets.TRE_ID }} + LOCATION: ${{ vars.LOCATION }} + AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT || 'AzureCloud' }} + run: python3 e2e_tests/run_bundle.py --prepare + + - name: Run E2E Tests + uses: ./.github/actions/devcontainer_run_command + with: + COMMAND: >- + TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS=${{ inputs.runCertificateTestsOnWeekends }} + TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle + DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} + CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_ENVIRONMENT: ${{ vars.AZURE_ENVIRONMENT }} + LOCATION: ${{ vars.LOCATION }} + API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}" + AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}" + TEST_APP_ID: "${{ secrets.TEST_APP_ID }}" + TEST_WORKSPACE_APP_ID: "${{ secrets.TEST_WORKSPACE_APP_ID }}" + TEST_WORKSPACE_APP_SECRET: "${{ secrets.TEST_WORKSPACE_APP_SECRET }}" + TEST_ACCOUNT_CLIENT_ID: "${{ secrets.TEST_ACCOUNT_CLIENT_ID }}" + TEST_ACCOUNT_CLIENT_SECRET: "${{ secrets.TEST_ACCOUNT_CLIENT_SECRET }}" + TRE_ID: ${{ secrets.TRE_ID }} + IS_API_SECURED: false + WORKSPACE_APP_SERVICE_PLAN_SKU: ${{ vars.WORKSPACE_APP_SERVICE_PLAN_SKU }} + E2E_TESTS_NUMBER_PROCESSES: ${{ inputs.E2E_TESTS_NUMBER_PROCESSES }} + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: E2E Test Results (Bundle) + path: | + ./e2e_tests/pytest_e2e_*.xml + ./e2e_tests/bundle-validation.json + summary: name: Summary - needs: [e2e_tests_smoke, e2e_tests_plan, e2e_tests_custom] + needs: [e2e_tests_smoke, e2e_tests_plan, e2e_tests_custom, e2e_tests_bundle] runs-on: ubuntu-latest permissions: checks: write diff --git a/.github/workflows/e2e_helper_tests.yml b/.github/workflows/e2e_helper_tests.yml index 85a7be3431..ec664e5446 100644 --- a/.github/workflows/e2e_helper_tests.yml +++ b/.github/workflows/e2e_helper_tests.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy - "feature/**" paths: - 'e2e_tests/**' + - 'templates/**/porter.yaml' - '.github/workflows/e2e_helper_tests.yml' permissions: diff --git a/CHANGELOG.md b/CHANGELOG.md index 4defc98c0a..b8da604d70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: +* Select individual bundle tests, report coverage limits and run selected tests after successful smoke tests. ([#5256](https://github.com/microsoft/AzureTRE/pull/5256)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) * Allow numeric CIDR masks in `address_space_size` (e.g. "23") when requesting auto-assigned address spaces; accepts numeric strings and validates the mask range. (`API` 0.28.0, `tre-workspace-base` 2.12.0, `tre-workspace-unrestricted` 0.15.0, `tre-workspace-airlock-import-review` 0.18.0) ([#4733](https://github.com/microsoft/AzureTRE/issues/4733)) diff --git a/Makefile b/Makefile index ab061901b6..1b24069859 100644 --- a/Makefile +++ b/Makefile @@ -497,8 +497,7 @@ test-e2e-smoke: ## 🧪 Run E2E smoke tests # Description: Run E2E extended tests # # The E2E extended tests include: # # - test_create_workspace_templates: POST Request to create a workspace template -# # - test_create_guacamole_service_into_base_workspace: POST Request to create a workspace and a separate POST call to create a guacamole workspace service into the base workspace -# # - test_airlock_flow: test import and export flow +# # - test_create_guacamole_service_into_base_workspace: create Guacamole and a Windows VM # Example: make test-e2e-extended test-e2e-extended: ## 🧪 Run E2E extended tests $(call target_title, "Running E2E extended tests") && \ @@ -532,7 +531,7 @@ test-e2e-backups: ## 🧪 Run E2E backup tests # Description: Run E2E airlock tests # # The E2E airlock tests include: -# # - tests marked with the `airlock` selector that verify airlock import/export flows and their access controls +# # - tests marked with the `airlock` selector that verify import flows, access controls and export request storage routing # Example: make test-e2e-airlock test-e2e-airlock: ## 🧪 Run E2E airlock tests $(call target_title, "Running E2E airlock tests") && \ @@ -554,6 +553,15 @@ test-e2e-custom: ## 🧪 Run E2E tests with custom selector (SELECTOR=) else \ python -m pytest -n "${E2E_TESTS_NUMBER_PROCESSES_DEFAULT}" -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml "pytest_e2e_$${SELECTOR// /_}.xml"; fi +# Description: Run the exact cases declared for TEST_BUNDLE, or a validated CI request. +# Example: TEST_BUNDLE=tre-workspace-service-azuresql make test-e2e-bundle +.PHONY: test-e2e-bundle +test-e2e-bundle: ## Run one bundle's explicit E2E cases + python3 "${MAKEFILE_DIR}/e2e_tests/run_bundle.py" --validate-only \ + && . ${MAKEFILE_DIR}/devops/scripts/bootstrap_azure_env.sh \ + && . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${MAKEFILE_DIR}/e2e_tests/.env \ + && python3 "${MAKEFILE_DIR}/e2e_tests/run_bundle.py" + # Description: Execute exact cases selected for one CI job. test-e2e-ci-group: ## 🧪 Run one selected E2E CI group . ${MAKEFILE_DIR}/devops/scripts/bootstrap_azure_env.sh \ diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index cc9540e7d0..bdb62a9cc7 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -24,3 +24,87 @@ Use the "Run and Debug" panel within Visual Studio Code, select "E2E Extended", - This will copy `config.yaml` settings to `/workspaces/AzureTRE/e2e_tests/.env` for you which supplies your authentication details - This will also use `/workspaces/AzureTRE/core/private.env` file for other values. + +## Validate one bundle + +The [bundle coverage map](https://github.com/microsoft/AzureTRE/blob/main/e2e_tests/bundle_coverage.json) lists every Porter bundle, its prerequisite bundles, exact pytest cases and remaining coverage gaps. +Selecting a bundle runs only its declared cases. Fixtures may deploy prerequisite resources. +A passing case does not prove the bundle's full functional, upgrade or custom-action lifecycle. + +### Check selection without deploying + +Use the E2E Python environment from the repository root: + +```bash +python3 e2e_tests/run_bundle.py --bundle tre-workspace-service-azuresql --collect-only +``` + +This collects the named Azure SQL parametrisation without creating resources. +The report records `not_run`, even when collection succeeds. +Unknown names, unavailable cases, missing tests and unexpected collected cases fail explicitly. + +### Run against a configured environment + +After configuring the prerequisites above, run: + +```bash +TEST_BUNDLE=tre-workspace-service-azuresql make test-e2e-bundle +``` + +Use one test process. Per-bundle runs reject a marker expression or multiple workers. +If the selection includes the Guacamole AAD case, clear `TEST_AAD_WORKSPACE_ID`. +That case requires an owned workspace so cascade cleanup removes its new service. +Some shared-service cases replace existing services, so use an isolated validation environment. +Check the selected entry's prerequisites and limitations before running it. + +### Certificate and Nexus selection + +Select `tre-shared-service-certs` to create and delete only the certificate shared service. +The case checks the deployed API record and confirms HTTP 404 after deletion. It does not require Nexus EULA consent. +It refuses to change resources when Nexus is present. Existing certificate resources must belong to earlier E2E tests before recovery can remove them. +Cleanup has a separate deadline and retains the certificate service if Nexus appears before deletion. +Use an isolated environment and run mutating suites sequentially. These guards do not provide a distributed lock. + +Select `tre-shared-service-sonatype-nexus` to run the existing certificate/Nexus lifecycle case. +The report identifies certificates and the deployed firewall as prerequisites. Dependency creation does not complete their separate evidence records. +Nexus selection still requires explicit EULA consent. Both selections retain the weekend certificate rate-limit precaution. +After checking rate limits, use `runCertificateTestsOnWeekends=true` in the branch workflow or `TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS=true` locally. +A skipped case remains unproven coverage. + +The certificate-only case is opt-in through bundle selection or the `certificate_validation` marker. +It is excluded from the ordinary `shared_services` selection to avoid an additional certificate request in that suite. +The existing combined case remains in that suite. It deletes Nexus before its certificate dependency. +Certificate content, renewal, retained Key Vault artefacts, Nexus package/proxy use, upgrades and physical resource removal need separate evidence. + +### Run the branch workflow before merge + +1. Push the reviewed changes to a branch in `microsoft/AzureTRE`. +2. Select that branch in **Deploy Azure TRE (branch)**. +3. Set `e2eBundle` to the exact Porter bundle name. +4. Leave `e2eTestsCustomSelector` empty and set `e2eProcesses` to `1`. +5. Keep `skipDeployment` false for a new branch environment. + +The workflow validates selection before deployment. It passes the bundle name as data in a JSON request, rather than embedding it in shell code. +Prepared requests require every generated metadata field, a full checkout SHA and non-empty TRE, location and cloud identifiers. +The test runner rejects incomplete requests and mismatches with the current checkout or configured TRE, location and cloud. +Unknown fields are ignored. Workflow metadata may be empty outside GitHub Actions. +Branch and PR-comment references derive different environment IDs. A completed test run does not remove its core environment. + +Bundle tests run in a separate job. Marker selections use the existing sequential test groups. +Both paths publish their results to the workflow summary. + +Existing slash commands retain their marker-based behaviour. They load workflow definitions from `main`; new branch-workflow inputs do not require merging first. + +### Read the evidence + +The **E2E Test Results (Bundle)** artefact contains: + +- `pytest_e2e_bundle.xml`: test, failure, skip and teardown results. +- `bundle-validation.json`: requested bundle, source version, declared prerequisite versions, requested and collected cases, coverage limitations, commit/workflow provenance, environment identity and observed resource/operation IDs. + +The JSON report copies only selected identifiers and operation states. It excludes resource properties, tokens and failure tracebacks. +Source versions describe the checked-out manifests; verify deployed versions separately when reusing an environment. +`passed_selected_cases` means the selected assertions and their teardown passed. The report always leaves `full_lifecycle_proven` false for a separate evidence review. +An entirely skipped selection exits unsuccessfully. Individual skipped cases retain their reasons, and a mixed pass/skip result is marked `partial`. + +Unavailable bundles fail with the gap recorded in the map. Adding a selector does not implement missing tests or fix bundle publication and registration gaps. diff --git a/docs/tre-developers/github-pr-bot-commands.md b/docs/tre-developers/github-pr-bot-commands.md index 5fd03858ee..edda0af0cc 100644 --- a/docs/tre-developers/github-pr-bot-commands.md +++ b/docs/tre-developers/github-pr-bot-commands.md @@ -8,6 +8,35 @@ These commands can only be run when commented by a user who is identified as a repo collaborator (see [granting access to run commands](#granting-access-to-run-commands)) +### Test scope + +Each test command builds and deploys the environment and runs smoke tests, unless the documented exception applies. +The table describes the selected assertions. A passing command does not prove every bundle or lifecycle action. + +| Command | Selected tests | Coverage limit | +| --- | --- | --- | +| `/test` | API health and template checks | No bundle lifecycle proof. Documentation-only changes receive a waiver. | +| `/test-extended` | Guacamole with a Windows VM, unrestricted workspace and import review workspace | Three selected cases. The suite excludes Linux VMs and the general workspace-service cases. | +| `/test-extended-aad` | Guacamole deployment and authentication redirect in an automatically configured Entra workspace | No full identity or permissions audit. | +| `/test-shared-services` | Firewall changes and restoration, selected service lifecycles, and the combined certificate/Nexus case | Nexus requires EULA consent. A skipped certificate case provides no lifecycle proof. | +| `/test-airlock` | Import requests, access checks, the import review VM flow and export request storage routing | No export review VM or migration proof. | +| `/test-backups` | Base workspace deployment with backups enabled and disabled | No backup restore or retention proof. | + +The `workspace_services`, `linux_vm` and `airlock_validation` markers have no dedicated slash command. +Use the branch workflow or the local test runner for these selections. +The `airlock_validation` marker selects two Airlock cases. It does not select the full `airlock` suite. +See [bundle selection](end-to-end-tests.md#validate-one-bundle) for exact cases and declared coverage gaps. + +### Environment and evidence + +Commands for the same Git reference share a deployment queue. Separate references can use separate environments. +Before parallel runs, check Azure capacity and shared dependencies. +PR-comment runs and branch-workflow runs use different environment identities. + +Record the checked-out commit, run URL, selected cases, skips and cleanup result. +When using `skip_deployment`, also record the deployed version. The test commit does not prove which version is deployed. +A waiver, successful build or successful collection is not an executed E2E result. + ### `/help` This command will cause the pr-comment-bot to respond with a comment listing the available commands. @@ -75,6 +104,8 @@ Push runs retain their existing extended/AAD selection. An excluded Nexus case p Shared-service tests allow one hour for prior-resource recovery, one hour for provisioning and one hour for each new resource's cleanup. Nexus and certificate recovery share the first deadline. Test watchdogs include a further 30-minute margin, including failed-create cleanup. +To permit the certificate case at weekends, set `runCertificateTestsOnWeekends=true` in the branch workflow. + The reusable workflow collects the selected cases and gives Nexus a separate five-hour job. The other cases run in their own job. These jobs run sequentially against the same environment, including when a group fails. Empty groups are omitted. Invalid or empty selections fail planning. Each job verifies the planned checkout and exact cases before execution. @@ -121,7 +152,7 @@ The `/test-destroy-env` command also destroys the environment associated with th ### `/test-force-approve` -This command skips running tests for a build and marks the checks as completed. +This command records a waiver and marks the checks as completed. It does not run tests. This is intended to be used in scenarios where running the tests for a PR doesn't add value (for example, changing a workflow file that is always pulled from the default branch). diff --git a/e2e_tests/.gitignore b/e2e_tests/.gitignore new file mode 100644 index 0000000000..df8a068fe0 --- /dev/null +++ b/e2e_tests/.gitignore @@ -0,0 +1,5 @@ + +bundle-validation.json +bundle-validation.json.tmp +bundle-validation-request.json +pytest_e2e_bundle.xml diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json new file mode 100644 index 0000000000..32a009dae0 --- /dev/null +++ b/e2e_tests/bundle_coverage.json @@ -0,0 +1,462 @@ +{ + "schema_version": 1, + "bundles": { + "tre-shared-service-admin-vm": { + "manifest": "templates/shared_services/admin-vm/porter.yaml", + "family": "shared_service", + "prerequisites": [], + "tests": [ + "test_shared_services.py::test_create_shared_service[tre-shared-service-admin-vm]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately.", + "The case recovers earlier test resources and preserves unrelated services. Use an isolated validation environment." + ], + "unavailable_reason": "" + }, + "tre-shared-service-airlock-notifier": { + "manifest": "templates/shared_services/airlock_notifier/porter.yaml", + "family": "shared_service", + "prerequisites": [], + "tests": [ + "test_shared_services.py::test_create_shared_service[tre-shared-service-airlock-notifier]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately.", + "The case recovers earlier test resources and preserves unrelated services. Use an isolated validation environment.", + "The case uses example SMTP settings and does not prove email delivery." + ], + "unavailable_reason": "" + }, + "tre-shared-service-certs": { + "manifest": "templates/shared_services/certs/porter.yaml", + "family": "shared_service", + "prerequisites": [ + "tre-shared-service-firewall" + ], + "tests": [ + "test_shared_services.py::test_create_certificate_shared_service" + ], + "related_tests": [ + "test_shared_services.py::test_create_certs_nexus_shared_service" + ], + "coverage": [ + "install", + "deployment-status", + "uninstall", + "api-removal" + ], + "limitations": [ + "Requires the deployed firewall. Uses an isolated environment and refuses to start while Nexus exists. Only earlier E2E certificate resources can be recovered.", + "Skips at weekends unless explicitly overridden after checking certificate rate limits. A skip does not prove the lifecycle.", + "Validates the API lifecycle, not certificate contents, renewal, retained Key Vault artefacts, upgrades or physical resource removal.", + "Certificate cleanup is retained if Nexus is present. Run mutating suites sequentially; the guard is not a distributed lock." + ], + "unavailable_reason": "" + }, + "tre-shared-service-cyclecloud": { + "manifest": "templates/shared_services/cyclecloud/porter.yaml", + "family": "shared_service", + "prerequisites": [], + "tests": [], + "related_tests": [], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "The existing shared-service case is disabled pending repair (#5154)." + }, + "tre-shared-service-databricks-private-auth": { + "manifest": "templates/shared_services/databricks-auth/porter.yaml", + "family": "shared_service", + "prerequisites": [], + "tests": [], + "related_tests": [], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "No dedicated shared-service lifecycle case (#5154)." + }, + "tre-shared-service-firewall": { + "manifest": "templates/shared_services/firewall/porter.yaml", + "family": "shared_service", + "prerequisites": [], + "tests": [ + "test_shared_services.py::test_patch_firewall" + ], + "related_tests": [], + "coverage": [ + "update" + ], + "limitations": [ + "Requires the deployed core firewall shared service. Temporarily replaces its display name and application rules, then restores both. Restoration failures fail the case; concurrent edits are not overwritten. Install, functional traffic checks and uninstall are not covered." + ], + "unavailable_reason": "" + }, + "tre-shared-service-gitea": { + "manifest": "templates/shared_services/gitea/porter.yaml", + "family": "shared_service", + "prerequisites": [], + "tests": [ + "test_shared_services.py::test_create_shared_service[tre-shared-service-gitea]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately.", + "The case recovers earlier test resources and preserves unrelated services. Use an isolated validation environment." + ], + "unavailable_reason": "" + }, + "tre-shared-service-sonatype-nexus": { + "manifest": "templates/shared_services/sonatype-nexus-vm/porter.yaml", + "family": "shared_service", + "prerequisites": [ + "tre-shared-service-firewall", + "tre-shared-service-certs" + ], + "tests": [ + "test_shared_services.py::test_create_certs_nexus_shared_service" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Creates a certificate prerequisite and Nexus. Requires explicit Nexus EULA consent and the deployed firewall.", + "Skips at weekends unless explicitly overridden after checking certificate rate limits. A skip does not prove the lifecycle.", + "Recovers earlier E2E resources, preserves unrelated services, and retains certificates if Nexus cleanup fails. Use an isolated environment and sequential mutations.", + "Dependency creation does not complete separate certificate or firewall evidence. Package/proxy use, upgrades and physical resource removal need separate checks." + ], + "unavailable_reason": "" + }, + "tre-service-azureml": { + "manifest": "templates/workspace_services/azureml/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [ + "test_workspace_services.py::test_install_workspace_service[tre-service-azureml]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "" + }, + "tre-user-resource-aml-compute-instance": { + "manifest": "templates/workspace_services/azureml/user_resources/aml_compute/porter.yaml", + "family": "user_resource", + "prerequisites": [ + "tre-workspace-base", + "tre-service-azureml" + ], + "tests": [], + "related_tests": [], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "Bundle publication, registration and lifecycle coverage are missing (#5127)." + }, + "tre-workspace-service-azuresql": { + "manifest": "templates/workspace_services/azuresql/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [ + "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-azuresql]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "" + }, + "tre-service-databricks": { + "manifest": "templates/workspace_services/databricks/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [], + "related_tests": [], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "Missing workspace-service case and parametrisation (#5126)." + }, + "tre-workspace-service-gitea": { + "manifest": "templates/workspace_services/gitea/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [ + "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-gitea]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "" + }, + "tre-service-guacamole": { + "manifest": "templates/workspace_services/guacamole/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [ + "test_workspace_services.py::test_create_guacamole_service_into_aad_workspace" + ], + "related_tests": [], + "coverage": [ + "install", + "authentication-redirect", + "workspace-cascade-cleanup" + ], + "limitations": [ + "Requires an automatically provisioned Entra workspace. Independent service deletion and upgrades are not covered.", + "The AAD case requires a new workspace. A configured TEST_AAD_WORKSPACE_ID is rejected to preserve cleanup." + ], + "unavailable_reason": "" + }, + "tre-service-guacamole-export-reviewvm": { + "manifest": "templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml", + "family": "user_resource", + "prerequisites": [ + "tre-workspace-base", + "tre-service-guacamole", + "tre-shared-service-certs", + "tre-shared-service-sonatype-nexus" + ], + "tests": [], + "related_tests": [], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately.", + "Requires a suitable Nexus instance or explicit Nexus EULA consent to create prerequisites (#5182)." + ], + "unavailable_reason": "No isolated export-review VM lifecycle case (#5127)." + }, + "tre-service-guacamole-import-reviewvm": { + "manifest": "templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml", + "family": "user_resource", + "prerequisites": [ + "tre-workspace-base", + "tre-workspace-airlock-import-review", + "tre-service-guacamole", + "tre-shared-service-certs", + "tre-shared-service-sonatype-nexus" + ], + "tests": [ + "test_airlock.py::test_airlock_review_vm_flow" + ], + "related_tests": [], + "coverage": [ + "install", + "airlock-review-flow", + "automatic-uninstall" + ], + "limitations": [ + "The case checks the import review flow and waits for automatic VM deletion after approval.", + "Export review, migration and interactive desktop access need separate tests.", + "Use a suitable Nexus instance or give explicit Nexus EULA consent to create prerequisites." + ], + "unavailable_reason": "" + }, + "tre-service-guacamole-linuxvm": { + "manifest": "templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml", + "family": "user_resource", + "prerequisites": [ + "tre-workspace-base", + "tre-service-guacamole", + "tre-shared-service-certs", + "tre-shared-service-sonatype-nexus" + ], + "tests": [ + "test_guacamole_linuxvm.py::test_create_guacamole_linux_vm" + ], + "related_tests": [], + "coverage": [ + "install", + "bootstrap-completion", + "uninstall" + ], + "limitations": [ + "Requires a suitable Nexus instance or explicit Nexus EULA consent to create prerequisites. VM availability and interactive access need separate validation." + ], + "unavailable_reason": "" + }, + "tre-service-guacamole-windowsvm": { + "manifest": "templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml", + "family": "user_resource", + "prerequisites": [ + "tre-workspace-base", + "tre-service-guacamole" + ], + "tests": [ + "test_workspace_services.py::test_create_guacamole_service_into_base_workspace" + ], + "related_tests": [], + "coverage": [ + "install", + "workspace-cascade-cleanup" + ], + "limitations": [ + "The Guacamole redirect is checked, but an interactive VM connection and independent VM deletion are not covered." + ], + "unavailable_reason": "" + }, + "tre-workspace-service-health": { + "manifest": "templates/workspace_services/health-services/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [ + "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-health]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately.", + "Default inputs do not enable FHIR or DICOM. Endpoint validation requires an explicit configuration." + ], + "unavailable_reason": "" + }, + "tre-workspace-service-mysql": { + "manifest": "templates/workspace_services/mysql/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [ + "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-mysql]" + ], + "related_tests": [], + "coverage": [ + "install", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "" + }, + "tre-workspace-service-ohdsi": { + "manifest": "templates/workspace_services/ohdsi/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [], + "related_tests": [], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "Missing workspace-service case and parametrisation (#5126)." + }, + "tre-workspace-service-openai": { + "manifest": "templates/workspace_services/openai/porter.yaml", + "family": "workspace_service", + "prerequisites": [ + "tre-workspace-base" + ], + "tests": [], + "related_tests": [ + "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-openai]" + ], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "The branch deployment does not register this bundle (#5150)." + }, + "tre-workspace-airlock-import-review": { + "manifest": "templates/workspaces/airlock-import-review/porter.yaml", + "family": "workspace", + "prerequisites": [], + "tests": [ + "test_workspace_templates.py::test_create_workspace_templates[tre-workspace-airlock-import-review]" + ], + "related_tests": [], + "coverage": [ + "install", + "deployment-status", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "" + }, + "tre-workspace-base": { + "manifest": "templates/workspaces/base/porter.yaml", + "family": "workspace", + "prerequisites": [], + "tests": [], + "related_tests": [ + "test_workspace_services.py::test_create_guacamole_service_into_base_workspace" + ], + "coverage": [], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "Base workspace provisioning is fixture coverage in other bundle tests; an isolated lifecycle case is missing (#5152)." + }, + "tre-workspace-unrestricted": { + "manifest": "templates/workspaces/unrestricted/porter.yaml", + "family": "workspace", + "prerequisites": [], + "tests": [ + "test_workspace_templates.py::test_create_workspace_templates[tre-workspace-unrestricted]" + ], + "related_tests": [], + "coverage": [ + "install", + "deployment-status", + "uninstall" + ], + "limitations": [ + "Functional, upgrade and custom-action coverage must be assessed separately." + ], + "unavailable_reason": "" + } + } +} diff --git a/e2e_tests/bundle_evidence.py b/e2e_tests/bundle_evidence.py new file mode 100644 index 0000000000..ea44176e43 --- /dev/null +++ b/e2e_tests/bundle_evidence.py @@ -0,0 +1,117 @@ +"""Write bounded, secret-free evidence for an explicitly selected bundle run.""" + +import json +from pathlib import Path +from urllib.parse import urlsplit + +_ACTIVE_REPORT = None + + +def activate(report): + global _ACTIVE_REPORT + _ACTIVE_REPORT = report + + +def record_resource(payload, operation, method): + if _ACTIVE_REPORT is None: + return + record = {key: operation[key] for key in ("id", "resourceId", "resourcePath") if key in operation} + record["method"] = method + if payload and "templateName" in payload: + record["bundle"] = payload["templateName"] + if payload and "templateVersion" in payload: + record["requested_template_version"] = payload["templateVersion"] + # Properties and operation messages can contain credentials. Do not copy them. + _ACTIVE_REPORT.data["resources"].append(record) + _ACTIVE_REPORT.write() + + +def record_reused_resource(resource): + """Record only the selected prerequisite's identity and deployed version.""" + if _ACTIVE_REPORT is None: + return + record = {key: resource[key] for key in ("id", "templateName", "templateVersion") if key in resource} + if not record.get("id") or not record.get("templateName"): + raise ValueError("Selected prerequisite is missing its resource identity") + if record not in _ACTIVE_REPORT.data["reused_resources"]: + _ACTIVE_REPORT.data["reused_resources"].append(record) + _ACTIVE_REPORT.write() + + +def record_operation(endpoint, state, done): + if _ACTIVE_REPORT is None: + return + # Only retain the API path, never a host, token or query string. + key = urlsplit(endpoint).path + _ACTIVE_REPORT.data["operations"][key] = {"state": state, "done": done} + _ACTIVE_REPORT.write() + + +class BundleReport: + def __init__(self, filename, data): + self.filename = Path(filename) + self.data = { + **data, + "full_lifecycle_proven": False, + "collected_tests": [], + "test_results": [], + "resources": [], + "reused_resources": [], + "operations": {}, + } + self.expected_tests = set(data.get("requested_tests", [])) + self.write() + + def write(self): + self.filename.parent.mkdir(parents=True, exist_ok=True) + temporary = self.filename.with_suffix(self.filename.suffix + ".tmp") + temporary.write_text(json.dumps(self.data, indent=2) + "\n") + temporary.replace(self.filename) + + def pytest_collection_finish(self, session): + import pytest + + collected = [item.nodeid for item in session.items] + self.data["collected_tests"] = collected + self.write() + if not collected: + raise pytest.UsageError("Bundle selection collected zero tests") + if set(collected) != self.expected_tests: + raise pytest.UsageError("Collected tests do not match the explicit bundle coverage map") + + def pytest_runtest_logreport(self, report): + result = { + "nodeid": report.nodeid, + "phase": report.when, + "outcome": report.outcome, + "duration_seconds": report.duration, + } + if report.skipped: + result["reason"] = str(report.longrepr[2]) if isinstance(report.longrepr, tuple) else str(report.longrepr) + # Failure tracebacks may include request bodies or credentials. Keep them in + # the existing test logs rather than copying them into the coverage report. + self.data["test_results"].append(result) + self.write() + + def finish(self, exit_code, collect_only): + results = self.data["test_results"] + calls = [result for result in results if result["phase"] == "call"] + self.data["counts"] = { + "passed": sum(result["outcome"] == "passed" for result in calls), + "failed": sum(result["outcome"] == "failed" for result in calls), + "errors": sum(result["outcome"] == "failed" and result["phase"] != "call" for result in results), + "skipped": sum(result["outcome"] == "skipped" for result in results), + } + if collect_only: + state = "not_run" if exit_code == 0 else "collection_failed" + elif exit_code: + state = "failed" + elif not calls or not self.data["counts"]["passed"]: + state, exit_code = "skipped", 1 + elif self.data["counts"]["skipped"]: + state = "partial" + else: + state = "passed_selected_cases" + self.data.update(status=state, exit_code=int(exit_code), full_lifecycle_proven=False) + self.write() + return int(exit_code) diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index 10717ddfc3..866dcfdffc 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -5,6 +5,7 @@ markers = linux_vm: opt-in Guacamole Linux VM provisioning check requiring Nexus or explicit EULA consent to create it extended_aad shared_services + certificate_validation: opt-in certificate-only lifecycle without Nexus deployment nexus: certificate and Nexus lifecycle requiring explicit EULA consent nexus_required: tests that need an existing Nexus service or explicit EULA consent to create one backups: only backup related diff --git a/e2e_tests/resources/nexus.py b/e2e_tests/resources/nexus.py index 05acf9aa9f..dee7b802e7 100644 --- a/e2e_tests/resources/nexus.py +++ b/e2e_tests/resources/nexus.py @@ -5,6 +5,7 @@ from packaging.version import Version +from e2e_tests.bundle_evidence import record_reused_resource from e2e_tests import config from e2e_tests.helpers import get_admin_token from e2e_tests.resources import strings @@ -54,6 +55,14 @@ async def nexus_prerequisites(verify): "Nexus must be deployed and enabled" ) assert Version(nexus["templateVersion"]) >= Version("3.11.0"), "Upgrade Nexus to bundle 3.11.0 before this test" + record_reused_resource(nexus) + for service in services: + if ( + service["templateName"] == strings.CERTS_SHARED_SERVICE + and nexus.get("properties", {}).get("ssl_cert_name") + and service.get("properties", {}).get("cert_name") == nexus["properties"]["ssl_cert_name"] + ): + record_reused_resource(service) yield return @@ -73,6 +82,7 @@ async def nexus_prerequisites(verify): "The existing certificate service belongs to another domain. Deploy Nexus separately before this test." ) cert_name = certs["properties"]["cert_name"] + record_reused_resource(certs) else: cert_name = f"nexus-e2e-{uuid4().hex[:8]}" await resources.enter_async_context( diff --git a/e2e_tests/resources/resource.py b/e2e_tests/resources/resource.py index efea3c8f91..aa5fb7d91a 100644 --- a/e2e_tests/resources/resource.py +++ b/e2e_tests/resources/resource.py @@ -1,5 +1,6 @@ import asyncio import logging +from e2e_tests.bundle_evidence import record_resource, record_operation from contextlib import asynccontextmanager from e2e_tests.timeouts import cleanup_deadline from httpx import AsyncClient, Timeout @@ -53,6 +54,7 @@ async def post_resource( assert_status(response, [status.HTTP_202_ACCEPTED], "The resource couldn't be sent") + record_resource(payload, response.json()["operation"], method) resource_path = response.json()["operation"]["resourcePath"] resource_id = response.json()["operation"]["resourceId"] operation_endpoint = response.headers["Location"] @@ -165,10 +167,12 @@ async def wait_for(func, client, operation_endpoint, access_token, failure_state LOGGER.info(f"WAITING FOR OP: {operation_endpoint}") try: done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) + record_operation(operation_endpoint, done_state, done) while not done: await asyncio.sleep(30) done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) + record_operation(operation_endpoint, done_state, done) LOGGER.info(f"{done}, {done_state}, {message}") except asyncio.CancelledError: LOGGER.error( diff --git a/e2e_tests/run_bundle.py b/e2e_tests/run_bundle.py new file mode 100644 index 0000000000..2fd28b461d --- /dev/null +++ b/e2e_tests/run_bundle.py @@ -0,0 +1,229 @@ +"""Select exact pytest cases from the Porter bundle coverage map.""" + +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +REPO_ROOT = Path(__file__).resolve().parents[1] +E2E_ROOT = REPO_ROOT / "e2e_tests" +CATALOG = E2E_ROOT / "bundle_coverage.json" +REQUEST_FILE = E2E_ROOT / "bundle-validation-request.json" +DEFAULT_REPORT = E2E_ROOT / "bundle-validation.json" +BUNDLE_NAME = re.compile(r"tre-[a-z0-9]+(?:-[a-z0-9]+)*\Z") +REQUEST_STRING_FIELDS = ( + "bundle", + "marker", + "processes", + "requested_checkout_sha", + "workflow_sha", + "workflow_ref", + "ci_ref", + "tre_id", + "location", + "cloud", + "run_id", + "run_attempt", +) + + +def checkout_sha(): + return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO_ROOT, text=True).strip() + + +def manifest_metadata(filename): + text = filename.read_text() + values = {} + for field in ("name", "version"): + match = re.search(rf"^{field}:\s*([a-zA-Z0-9.+-]+)\s*$", text, re.MULTILINE) + if not match: + raise ValueError(f"Missing plain scalar {field} in {filename.relative_to(REPO_ROOT)}") + values[field] = match[1] + return values + + +def load_catalog(): + catalog = json.loads(CATALOG.read_text()) + if catalog.get("schema_version") != 1: + raise ValueError("Unsupported bundle coverage schema") + bundles = catalog["bundles"] + manifests = {} + for filename in (REPO_ROOT / "templates").rglob("porter.yaml"): + metadata = manifest_metadata(filename) + if metadata["name"] in manifests: + raise ValueError(f"Duplicate Porter bundle name: {metadata['name']}") + manifests[metadata["name"]] = (filename, metadata["version"]) + if set(bundles) != set(manifests): + raise ValueError("Bundle coverage map does not match the current Porter bundle inventory") + for name, entry in bundles.items(): + filename, version = manifests[name] + if entry["manifest"] != str(filename.relative_to(REPO_ROOT)): + raise ValueError(f"Incorrect manifest path for {name}") + if not set(entry["prerequisites"]) <= bundles.keys(): + raise ValueError(f"Unknown prerequisite for {name}") + if bool(entry["tests"]) == bool(entry["unavailable_reason"]): + raise ValueError(f"Declare either executable cases or an unavailable reason for {name}") + for nodeid in entry["tests"] + entry["related_tests"]: + if not re.fullmatch(r"test_[a-z0-9_]+\.py::test_[a-z0-9_]+(?:\[[a-zA-Z0-9_-]+\])?", nodeid): + raise ValueError(f"Invalid explicit pytest node ID for {name}") + if not (E2E_ROOT / nodeid.split("::")[0]).is_file(): + raise ValueError(f"Missing test module for {name}") + entry["source_version"] = version + return bundles + + +def selection(bundle, marker="", processes="1"): + bundles = load_catalog() + if not BUNDLE_NAME.fullmatch(bundle) or bundle not in bundles: + raise ValueError(f"Unknown bundle identifier: {bundle!r}") + if marker: + raise ValueError("Select either a bundle or a marker expression, not both") + if str(processes) != "1": + raise ValueError("Per-bundle validation requires exactly one test process") + entry = bundles[bundle] + if not entry["tests"]: + raise ValueError(f"Bundle case is unavailable: {entry['unavailable_reason']}") + return entry, {name: bundles[name]["source_version"] for name in entry["prerequisites"]} + + +def request_from_environment(bundle): + return { + "bundle": bundle, + "marker": os.environ.get("TEST_BUNDLE_MARKER", ""), + "processes": os.environ.get("E2E_TESTS_NUMBER_PROCESSES", "1") or "1", + "requested_checkout_sha": checkout_sha(), + "workflow_sha": os.environ.get("GITHUB_WORKFLOW_SHA", ""), + "workflow_ref": os.environ.get("GITHUB_WORKFLOW_REF", ""), + "ci_ref": os.environ.get("TEST_BUNDLE_CI_REF", ""), + "tre_id": os.environ.get("TRE_ID", ""), + "location": os.environ.get("LOCATION", ""), + "cloud": os.environ.get("AZURE_ENVIRONMENT", ""), + "run_id": os.environ.get("GITHUB_RUN_ID", ""), + "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", ""), + "accept_nexus_eula": os.environ.get("TEST_ACCEPT_NEXUS_EULA", "false").lower() == "true", + } + + +def validate_prepared_request(prepared): + """Require the complete handoff without substituting current-process values.""" + if not isinstance(prepared, dict): + raise ValueError("A prepared request must be a JSON object") + for field in (*REQUEST_STRING_FIELDS, "accept_nexus_eula"): + if field not in prepared: + raise ValueError(f"Prepared request is missing required field: {field}") + for field in REQUEST_STRING_FIELDS: + if not isinstance(prepared[field], str): + raise ValueError(f"Prepared request field {field} must be a string") + if type(prepared["accept_nexus_eula"]) is not bool: + raise ValueError("Nexus consent in a prepared request must be a boolean") + if not re.fullmatch(r"[0-9a-f]{40}", prepared["requested_checkout_sha"]): + raise ValueError("Prepared request must contain a full checkout SHA") + for field in ("tre_id", "location", "cloud"): + if not prepared[field].strip(): + raise ValueError(f"Prepared request field {field} must not be empty") + # Unknown fields are excluded from the non-secret evidence report. + return {field: prepared[field] for field in (*REQUEST_STRING_FIELDS, "accept_nexus_eula")} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", default=os.environ.get("TEST_BUNDLE", "")) + parser.add_argument("--validate-only", action="store_true") + parser.add_argument("--prepare", action="store_true", help="Write the validated CI request for the devcontainer") + parser.add_argument("--collect-only", action="store_true") + parser.add_argument("--report", type=Path, default=DEFAULT_REPORT) + args = parser.parse_args(argv) + args.report = args.report.resolve() + sys.path.insert(0, str(REPO_ROOT)) + from e2e_tests.bundle_evidence import BundleReport, activate + + request = request_from_environment(args.bundle) + evidence = BundleReport(args.report, {"schema_version": 1, "status": "preflight", "requested_bundle": args.bundle}) + try: + filename = os.environ.get("TEST_BUNDLE_REQUEST_FILE", "") + if filename: + if args.bundle or args.prepare: + raise ValueError("A prepared request cannot be combined with another bundle selection") + request = validate_prepared_request(json.loads(Path(filename).read_text())) + if request["requested_checkout_sha"] != checkout_sha(): + raise ValueError("Prepared request belongs to a different checkout") + for field, variable in (("tre_id", "TRE_ID"), ("location", "LOCATION"), ("cloud", "AZURE_ENVIRONMENT")): + if os.environ.get(variable) and request[field] != os.environ[variable]: + raise ValueError(f"Prepared request belongs to a different TRE environment ({field})") + elif args.prepare: + request = validate_prepared_request(request) + entry, prerequisites = selection(request["bundle"], request["marker"], request["processes"]) + evidence.data.update( + requested_bundle=request["bundle"], + source_bundle_version=entry["source_version"], + manifest=entry["manifest"], + requested_tests=entry["tests"], + declared_prerequisites=prerequisites, + coverage=entry["coverage"], + limitations=entry["limitations"], + provenance=request, + checkout_sha=checkout_sha(), + ) + evidence.expected_tests = set(entry["tests"]) + evidence.write() + if args.prepare: + REQUEST_FILE.write_text(json.dumps(request, indent=2) + "\n") + if args.validate_only or args.prepare: + evidence.data["status"] = "not_run" + evidence.write() + print(f"Validated {request['bundle']}: {len(entry['tests'])} explicit test case(s)") + return 0 + os.environ["TEST_ACCEPT_NEXUS_EULA"] = "true" if request["accept_nexus_eula"] else "false" + os.chdir(E2E_ROOT) + sys.path.insert(0, str(E2E_ROOT)) + import pytest + import config as tre_config + + evidence.data["configured_resource_ids"] = { + name: getattr(tre_config, name) + for name in ( + "TEST_WORKSPACE_ID", + "TEST_WORKSPACE_SERVICE_ID", + "TEST_AAD_WORKSPACE_ID", + "TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_ID", + "TEST_AIRLOCK_IMPORT_REVIEW_WORKSPACE_SERVICE_ID", + ) + if getattr(tre_config, name) + } + aad_case = "test_workspace_services.py::test_create_guacamole_service_into_aad_workspace" + if tre_config.TEST_AAD_WORKSPACE_ID and aad_case in entry["tests"]: + raise ValueError( + "The selected AAD case requires a new workspace for service cleanup. " + "Clear TEST_AAD_WORKSPACE_ID before running this selection." + ) + activate(evidence) + evidence.data["status"] = "collecting" if args.collect_only else "running" + evidence.write() + pytest_args = [ + *entry["tests"], + "-o", + "addopts=", + "--junitxml=pytest_e2e_bundle.xml", + "--verify", + os.environ.get("IS_API_SECURED", "true"), + ] + if args.collect_only: + pytest_args.append("--collect-only") + # Ignore inherited addopts: -k, -m or xdist must not alter this exact selection. + os.environ.pop("PYTEST_ADDOPTS", None) + exit_code = pytest.main(pytest_args, plugins=[evidence]) + return evidence.finish(exit_code, args.collect_only) + except (ValueError, OSError, KeyError, TypeError) as error: + evidence.data.update(status="selection_error", error=str(error), exit_code=2, full_lifecycle_proven=False) + evidence.write() + print(f"Bundle validation failed: {error}", file=sys.stderr) + return 2 + finally: + activate(None) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index 1d8a95f03d..eae55f46fd 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -4,6 +4,8 @@ import pytest import logging +from httpx import AsyncClient +from starlette import status from e2e_tests import config from e2e_tests.conftest import disable_and_delete_tre_resource from datetime import date @@ -17,11 +19,13 @@ from helpers import get_shared_service_by_name from resources import strings from helpers import get_admin_token +from helpers import TIMEOUT, get_auth_header, get_full_endpoint LOGGER = logging.getLogger(__name__) RECOVERY_TIMEOUT_SECONDS = 60 * 60 PROVISIONING_TIMEOUT_SECONDS = 60 * 60 CLEANUP_TIMEOUT_SECONDS = 60 * 60 +CERTIFICATE_CHECK_TIMEOUT_SECONDS = 5 * 60 # Recover old resources under one shared deadline. Failed creation can spend # another cleanup deadline inside post_resource before unwinding the exit stack. SINGLE_SERVICE_TIMEOUT_SECONDS = ( @@ -225,6 +229,49 @@ async def test_create_shared_service(template_name, verify) -> None: await resources.enter_async_context(managed_shared_service(post_payload, verify)) +@pytest.mark.certificate_validation +@pytest.mark.timeout(SINGLE_SERVICE_TIMEOUT_SECONDS) +async def test_create_certificate_shared_service(verify) -> None: + if date.today().weekday() in [5, 6] and not config.TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS: + pytest.skip("Certificate rate-limit precaution: skipping on weekends; certificate coverage is unproven.") + + async with asyncio.timeout(RECOVERY_TIMEOUT_SECONDS): + admin_token = await get_admin_token(verify) + if await get_shared_service_by_name(strings.NEXUS_SHARED_SERVICE, verify, admin_token): + pytest.fail("Nexus is present. Certificate-only validation requires an isolated environment.") + await recover_previous_shared_services(verify, strings.CERTS_SHARED_SERVICE) + + payload = { + "templateName": strings.CERTS_SHARED_SERVICE, + "properties": { + "display_name": f"Shared service {strings.CERTS_SHARED_SERVICE}", + "description": f"{strings.CERTS_SHARED_SERVICE} deployed via e2e tests", + "domain_prefix": "nexus", + "cert_name": "nexus-ssl", + }, + } + async with AsyncExitStack() as resources: + async with asyncio.timeout(PROVISIONING_TIMEOUT_SECONDS): + resource_path = await resources.enter_async_context(managed_shared_service(payload, verify)) + async with asyncio.timeout(CERTIFICATE_CHECK_TIMEOUT_SECONDS): + certificate = (await get_resource(f"/api{resource_path}", admin_token, verify))["sharedService"] + assert certificate["id"] == resource_path.rsplit("/", 1)[-1] + assert certificate["templateName"] == strings.CERTS_SHARED_SERVICE + assert certificate["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED + assert certificate["isEnabled"] is True + assert certificate["properties"]["domain_prefix"] == "nexus" + assert certificate["properties"]["cert_name"] == "nexus-ssl" + + async with asyncio.timeout(CERTIFICATE_CHECK_TIMEOUT_SECONDS): + async with AsyncClient(verify=verify) as client: + response = await client.get( + get_full_endpoint(f"/api{resource_path}"), headers=get_auth_header(admin_token), timeout=TIMEOUT + ) + assert response.status_code == status.HTTP_404_NOT_FOUND, ( + f"Certificate service remains visible after uninstall: HTTP {response.status_code}" + ) + + @pytest.mark.shared_services @pytest.mark.nexus @pytest.mark.timeout(NEXUS_TIMEOUT_SECONDS) diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py index 92b7b0003c..fbb4704278 100644 --- a/e2e_tests/test_workspace_services.py +++ b/e2e_tests/test_workspace_services.py @@ -10,7 +10,6 @@ workspace_services = [ strings.AZUREML_SERVICE, strings.GITEA_SERVICE, - strings.MLFLOW_SERVICE, strings.MYSQL_SERVICE, strings.HEALTH_SERVICE, strings.AZURESQL_SERVICE, diff --git a/e2e_tests/unit_tests/README.md b/e2e_tests/unit_tests/README.md index 01f239cb5b..14fb1700d6 100644 --- a/e2e_tests/unit_tests/README.md +++ b/e2e_tests/unit_tests/README.md @@ -18,3 +18,7 @@ token. Literal token strings remain supported but cannot be renewed. These tests check the request and renewal behaviour locally. A live extended test run is still needed to validate the fix against Azure. + +The bundle-selector tests also collect the real Azure SQL case without running +fixtures. Small isolated pytest runs check exact parameter selection, skipped +cases, teardown failures and artefact redaction. No Azure deployment is required. diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py new file mode 100644 index 0000000000..e69e9d1f8f --- /dev/null +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -0,0 +1,417 @@ +"""Exercise the bundle selector and evidence boundary without Azure access.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest +from unittest.mock import patch + +from e2e_tests import bundle_evidence, run_bundle + + +ROOT = run_bundle.REPO_ROOT +SQL = "tre-workspace-service-azuresql" +SQL_CASE = f"test_workspace_services.py::test_install_workspace_service[{SQL}]" + + +class BundleSelectionTests(unittest.TestCase): + def prepared_request(self): + request = run_bundle.request_from_environment(SQL) + request.update(tre_id="tre-test", location="switzerlandnorth", cloud="AzureCloud") + return request + + def run_prepared_request(self, prepared, *arguments, environment=None): + with tempfile.TemporaryDirectory() as directory: + filename = Path(directory) / "request.json" + filename.write_text(json.dumps(prepared)) + return self.run_selector( + *arguments, environment={"TEST_BUNDLE_REQUEST_FILE": str(filename), **(environment or {})} + ) + + def run_selector(self, *arguments, environment=None, relative_report=False): + with tempfile.TemporaryDirectory() as directory: + report = Path(directory) / "result.json" + env = os.environ.copy() + for key in ( + "TEST_BUNDLE", + "TEST_BUNDLE_REQUEST_FILE", + "TEST_BUNDLE_MARKER", + "E2E_TESTS_NUMBER_PROCESSES", + "TRE_ID", + "LOCATION", + "AZURE_ENVIRONMENT", + "TEST_AAD_WORKSPACE_ID", + ): + env.pop(key, None) + env.update(environment or {}) + result = subprocess.run( + [ + sys.executable, + str(ROOT / "e2e_tests/run_bundle.py"), + "--report", + report.name if relative_report else str(report), + *arguments, + ], + cwd=directory if relative_report else ROOT, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertTrue(report.exists(), result.stdout + result.stderr) + return result, json.loads(report.read_text()) + + def test_catalog_covers_the_actual_porter_inventory(self): + catalog = run_bundle.load_catalog() + manifests = {entry["manifest"] for entry in catalog.values()} + self.assertEqual(manifests, {str(path.relative_to(ROOT)) for path in (ROOT / "templates").rglob("porter.yaml")}) + for name, entry in catalog.items(): + with self.subTest(bundle=name): + self.assertTrue(entry["tests"] or entry["unavailable_reason"]) + self.assertTrue(entry["source_version"]) + self.assertTrue(entry["limitations"]) + + def test_every_declared_case_collects_from_current_source(self): + expected = sorted({node for entry in run_bundle.load_catalog().values() for node in entry["tests"]}) + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(ROOT / "e2e_tests"))) + env.pop("PYTEST_ADDOPTS", None) + result = subprocess.run( + [sys.executable, "-m", "pytest", "--collect-only", "-q", "-o", "addopts=", *expected], + cwd=ROOT / "e2e_tests", + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + collected = sorted(line for line in result.stdout.splitlines() if line.startswith("test_") and "::" in line) + self.assertEqual(collected, expected) + + def test_sql_collects_only_its_named_parametrised_case(self): + result, report = self.run_selector( + "--bundle", SQL, "--collect-only", environment={"TEST_WORKSPACE_ID": "existing-workspace-id"} + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], [SQL_CASE]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + self.assertEqual(report["configured_resource_ids"], {"TEST_WORKSPACE_ID": "existing-workspace-id"}) + self.assertIn("tre-workspace-base", report["declared_prerequisites"]) + + def test_relative_report_reaches_final_status_in_the_caller_directory(self): + result, report = self.run_selector("--bundle", SQL, "--collect-only", relative_report=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], [SQL_CASE]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + + def test_aad_selection_rejects_a_reused_workspace_before_collection(self): + result, report = self.run_selector( + "--bundle", + "tre-service-guacamole", + "--collect-only", + environment={"TEST_AAD_WORKSPACE_ID": "existing-aad-workspace"}, + ) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertEqual(report["status"], "selection_error") + self.assertEqual(report["collected_tests"], []) + self.assertEqual(report["resources"], []) + self.assertIn("Clear TEST_AAD_WORKSPACE_ID", report["error"]) + + def test_unrelated_selection_does_not_reject_a_configured_aad_workspace(self): + result, report = self.run_selector( + "--bundle", + SQL, + "--collect-only", + environment={"TEST_AAD_WORKSPACE_ID": "existing-aad-workspace"}, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], [SQL_CASE]) + + def test_unknown_partial_and_shell_like_identifiers_fail_before_collection(self): + for identifier in ("azuresql", "tre-nonexistent", SQL + "; echo unexpected", SQL + "$(echo unexpected)"): + with self.subTest(identifier=identifier): + result, report = self.run_selector("--bundle", identifier, "--validate-only") + self.assertEqual(result.returncode, 2) + self.assertIn("Unknown bundle identifier", result.stderr) + self.assertEqual(report["status"], "selection_error") + self.assertEqual(report["collected_tests"], []) + + def test_certificate_and_nexus_selections_keep_target_and_dependency_evidence_separate(self): + firewall = "tre-shared-service-firewall" + certificate = "tre-shared-service-certs" + cases = ( + (certificate, "test_create_certificate_shared_service", {firewall}), + ("tre-shared-service-sonatype-nexus", "test_create_certs_nexus_shared_service", {firewall, certificate}), + ) + for bundle, case, prerequisites in cases: + with self.subTest(bundle=bundle): + result, report = self.run_selector("--bundle", bundle, "--collect-only") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_shared_services.py::" + case]) + self.assertEqual(set(report["declared_prerequisites"]), prerequisites) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + + def test_shared_service_suite_does_not_add_another_certificate_request(self): + env = dict(os.environ, PYTHONPATH=os.pathsep.join((str(ROOT), str(ROOT / "e2e_tests")))) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "-o", + "addopts=", + "-m", + "shared_services", + "test_shared_services.py", + ], + cwd=ROOT / "e2e_tests", + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + collected = [line for line in result.stdout.splitlines() if line.startswith("test_shared_services.py::")] + self.assertEqual(len(collected), 5) + self.assertIn("test_shared_services.py::test_create_certs_nexus_shared_service", collected) + self.assertNotIn("test_shared_services.py::test_create_certificate_shared_service", collected) + + def test_unavailable_bundle_fails_with_its_coverage_gap(self): + result, report = self.run_selector("--bundle", "tre-user-resource-aml-compute-instance", "--validate-only") + self.assertEqual(result.returncode, 2) + self.assertIn("#5127", report["error"]) + self.assertEqual(report["collected_tests"], []) + + def test_marker_and_parallel_worker_inputs_are_rejected(self): + for extra in ({"TEST_BUNDLE_MARKER": "workspace_services"}, {"E2E_TESTS_NUMBER_PROCESSES": "2"}): + with self.subTest(environment=extra): + result, report = self.run_selector("--bundle", SQL, "--validate-only", environment=extra) + self.assertEqual(result.returncode, 2) + self.assertEqual(report["status"], "selection_error") + + def test_prepared_request_must_match_checkout_and_environment(self): + cases = ( + ({"requested_checkout_sha": "0" * 40}, {}, "different checkout"), + ({"tre_id": "another-tre"}, {"TRE_ID": "expected-tre"}, "different TRE"), + ({"location": "eastus"}, {"LOCATION": "switzerlandnorth"}, "different TRE"), + ({"cloud": "AzureUSGovernment"}, {"AZURE_ENVIRONMENT": "AzureCloud"}, "different TRE"), + ({"accept_nexus_eula": "false"}, {}, "must be a boolean"), + ) + for change, extra_env, message in cases: + with self.subTest(change=change): + prepared = self.prepared_request() + prepared.update(change) + result, report = self.run_prepared_request(prepared, "--validate-only", environment=extra_env) + self.assertEqual(result.returncode, 2) + self.assertIn(message, report["error"]) + + def test_missing_request_fields_are_not_filled_from_the_current_environment(self): + complete = self.prepared_request() + for field in complete: + with self.subTest(field=field): + prepared = {key: value for key, value in complete.items() if key != field} + result, report = self.run_prepared_request( + prepared, + "--collect-only", + environment={ + "TRE_ID": "tre-test", + "LOCATION": "switzerlandnorth", + "AZURE_ENVIRONMENT": "AzureCloud", + }, + ) + self.assertEqual(result.returncode, 2) + self.assertIn(f"missing required field: {field}", report["error"]) + self.assertEqual(report["collected_tests"], []) + + def test_invalid_prepared_request_shape_and_types_fail_before_collection(self): + for prepared in (None, [], "invalid", 1): + with self.subTest(prepared=prepared): + result, report = self.run_prepared_request(prepared, "--collect-only") + self.assertEqual(result.returncode, 2) + self.assertIn("JSON object", report["error"]) + self.assertEqual(report["collected_tests"], []) + for field in run_bundle.REQUEST_STRING_FIELDS: + with self.subTest(field=field): + prepared = {**self.prepared_request(), field: None} + result, report = self.run_prepared_request(prepared, "--collect-only") + self.assertEqual(result.returncode, 2) + self.assertIn(f"{field} must be a string", report["error"]) + self.assertEqual(report["collected_tests"], []) + + def test_prepared_request_requires_checkout_and_environment_identifiers(self): + for field in ("requested_checkout_sha", "tre_id", "location", "cloud"): + with self.subTest(field=field): + prepared = {**self.prepared_request(), field: " "} + result, report = self.run_prepared_request(prepared, "--collect-only") + self.assertEqual(result.returncode, 2) + self.assertEqual(report["status"], "selection_error") + self.assertEqual(report["collected_tests"], []) + + def test_generated_request_round_trips_without_replacing_provenance(self): + with tempfile.TemporaryDirectory() as directory: + filename = Path(directory) / "request.json" + environment = { + "PATH": os.environ["PATH"], + "TRE_ID": "tre-test", + "LOCATION": "switzerlandnorth", + "AZURE_ENVIRONMENT": "AzureCloud", + "GITHUB_RUN_ID": "original-run", + } + with patch.dict(os.environ, environment, clear=True), patch.object(run_bundle, "REQUEST_FILE", filename): + code = run_bundle.main(["--bundle", SQL, "--prepare", "--report", str(Path(directory) / "report.json")]) + self.assertEqual(code, 0) + prepared = json.loads(filename.read_text()) + result, report = self.run_prepared_request( + prepared, "--collect-only", environment={"GITHUB_RUN_ID": "current-run"} + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(report["provenance"], prepared) + self.assertEqual(report["collected_tests"], [SQL_CASE]) + + def test_review_vm_inventory_includes_bootstrap_prerequisites(self): + catalog = run_bundle.load_catalog() + for bundle in ("tre-service-guacamole-import-reviewvm", "tre-service-guacamole-export-reviewvm"): + with self.subTest(bundle=bundle): + self.assertLessEqual( + {"tre-shared-service-certs", "tre-shared-service-sonatype-nexus"}, + set(catalog[bundle]["prerequisites"]), + ) + + def test_request_metadata_does_not_copy_unknown_secret_fields(self): + prepared = self.prepared_request() + prepared["secret_password"] = "never-copy-this-value" + result, report = self.run_prepared_request(prepared, "--validate-only") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("never-copy-this-value", json.dumps(report)) + self.assertEqual(report["requested_tests"], [SQL_CASE]) + + +class BundleEvidenceIntegrationTests(unittest.TestCase): + def run_pytest(self, source, expected, selection=None): + with tempfile.TemporaryDirectory() as directory: + folder = Path(directory) + (folder / "test_cases.py").write_text(textwrap.dedent(source)) + script = """ +import json +import pytest +from pathlib import Path +from e2e_tests.bundle_evidence import BundleReport +settings = json.loads(Path("settings.json").read_text()) +report = BundleReport("evidence.json", {"requested_tests": settings["expected"]}) +code = pytest.main(["-q", *settings["selection"]], plugins=[report]) +raise SystemExit(report.finish(code, False)) +""" + (folder / "runner.py").write_text(textwrap.dedent(script)) + (folder / "settings.json").write_text( + json.dumps({"expected": expected, "selection": selection or expected}) + ) + env = os.environ.copy() + env.update(PYTHONPATH=str(ROOT), PYTEST_DISABLE_PLUGIN_AUTOLOAD="1") + env.pop("PYTEST_ADDOPTS", None) + result = subprocess.run( + [sys.executable, "runner.py"], + cwd=folder, + env=env, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + self.assertTrue((folder / "evidence.json").exists(), result.stdout + result.stderr) + return result, json.loads((folder / "evidence.json").read_text()) + + def test_parameter_selection_does_not_execute_other_bundles(self): + source = """ +import pytest +@pytest.mark.parametrize("bundle", ["sql", "unrelated"]) +def test_bundle(bundle): + assert bundle == "sql", "an unrelated bundle was executed" +""" + node = "test_cases.py::test_bundle[sql]" + result, report = self.run_pytest(source, [node]) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], [node]) + self.assertEqual(report["counts"]["passed"], 1) + self.assertEqual(report["status"], "passed_selected_cases") + self.assertFalse(report["full_lifecycle_proven"]) + + def test_all_skipped_is_reported_and_does_not_pass(self): + result, report = self.run_pytest( + 'import pytest\n@pytest.mark.skip(reason="missing prerequisite")\ndef test_bundle(): pass\n', + ["test_cases.py::test_bundle"], + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(report["status"], "skipped") + self.assertEqual(report["counts"]["skipped"], 1) + self.assertIn("missing prerequisite", report["test_results"][0]["reason"]) + + def test_teardown_failure_is_not_a_bundle_pass(self): + source = """ +import pytest +@pytest.fixture +def resource(): + yield + raise RuntimeError("secret-in-existing-traceback") +def test_bundle(resource): pass +""" + result, report = self.run_pytest(source, ["test_cases.py::test_bundle"]) + self.assertEqual(result.returncode, 1) + self.assertEqual(report["status"], "failed") + self.assertEqual(report["counts"]["passed"], 1) + self.assertEqual(report["counts"]["errors"], 1) + self.assertNotIn("secret-in-existing-traceback", json.dumps(report)) + + def test_missing_or_extra_collected_cases_fail(self): + source = "def test_bundle(): pass\ndef test_other(): pass\n" + for requested in (["test_cases.py::test_missing"], ["test_cases.py"]): + with self.subTest(requested=requested): + result, report = self.run_pytest(source, ["test_cases.py::test_bundle"], requested) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(report["status"], "failed") + + def test_zero_collected_cases_fail(self): + result, report = self.run_pytest("", [], ["test_cases.py"]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(report["collected_tests"], []) + + def test_resource_journal_records_ids_without_properties_or_tokens(self): + with tempfile.TemporaryDirectory() as directory: + report = bundle_evidence.BundleReport(Path(directory) / "evidence.json", {}) + bundle_evidence.activate(report) + try: + bundle_evidence.record_resource( + {"templateName": SQL, "properties": {"password": "secret-value"}}, + { + "id": "op-id", + "resourceId": "sql-id", + "resourcePath": "/workspaces/ws/services/sql-id", + "message": "secret-value", + }, + "POST", + ) + bundle_evidence.record_operation( + "https://tre.example/api/operations/op-id?sig=secret-value", "deployed", True + ) + finally: + bundle_evidence.activate(None) + self.assertEqual(report.data["resources"][0]["resourceId"], "sql-id") + self.assertTrue(report.data["operations"]["/api/operations/op-id"]["done"]) + self.assertNotIn("secret-value", report.filename.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e_tests/unit_tests/test_certificate_selection.py b/e2e_tests/unit_tests/test_certificate_selection.py new file mode 100644 index 0000000000..232b716c73 --- /dev/null +++ b/e2e_tests/unit_tests/test_certificate_selection.py @@ -0,0 +1,163 @@ +"""Exercise certificate-only lifecycle boundaries without Azure access.""" + +import asyncio +import json +from pathlib import Path +import unittest +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from jsonschema import validate + +from e2e_tests import test_shared_services as shared + + +class CertificateLifecycleTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.date = self.enterContext(patch.object(shared, "date")) + self.date.today.return_value.weekday.return_value = 0 + self.enterContext(patch.object(shared.config, "TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS", False)) + self.enterContext(patch.object(shared.config, "TEST_ACCEPT_NEXUS_EULA", False)) + self.token = self.enterContext(patch.object(shared, "get_admin_token", AsyncMock(return_value="token"))) + self.find_service = self.enterContext( + patch.object(shared, "get_shared_service_by_name", AsyncMock(return_value=None)) + ) + self.recover = self.enterContext( + patch.object(shared, "disable_and_delete_shared_service_if_exists", AsyncMock()) + ) + self.post = self.enterContext( + patch.object(shared, "post_resource", AsyncMock(return_value=("/shared-services/certs", "certs"))) + ) + self.cleanup = self.enterContext(patch.object(shared, "disable_and_delete_tre_resource", AsyncMock())) + self.record = { + "id": "certs", + "templateName": shared.strings.CERTS_SHARED_SERVICE, + "deploymentStatus": "deployed", + "isEnabled": True, + "properties": {"domain_prefix": "nexus", "cert_name": "nexus-ssl"}, + } + self.get = self.enterContext( + patch.object(shared, "get_resource", AsyncMock(return_value={"sharedService": self.record})) + ) + self.lookup_status = 404 + self.lookups = [] + + def lookup(request): + self.assertEqual(request.method, "GET") + self.assertEqual(request.url.path, "/api/shared-services/certs") + self.assertEqual(request.headers["Authorization"], "Bearer token") + self.lookups.append(request) + return httpx.Response(self.lookup_status, request=request) + + self.enterContext( + patch.object( + shared, "AsyncClient", lambda **kwargs: httpx.AsyncClient(transport=httpx.MockTransport(lookup)) + ) + ) + self.enterContext(patch.object(shared, "get_full_endpoint", lambda endpoint: "https://tre.test" + endpoint)) + + async def test_creates_only_certificates_without_nexus_consent_and_confirms_removal(self): + await shared.test_create_certificate_shared_service(True) + self.post.assert_awaited_once() + arguments = self.post.await_args.kwargs + self.assertEqual(arguments["payload"]["templateName"], shared.strings.CERTS_SHARED_SERVICE) + self.assertTrue(arguments["cleanup_failed_create"]) + schema = Path(__file__).resolve().parents[2] / "templates/shared_services/certs/template_schema.json" + validate(arguments["payload"]["properties"], json.loads(schema.read_text())) + self.recover.assert_awaited_once_with(shared.strings.CERTS_SHARED_SERVICE, True) + self.cleanup.assert_awaited_once_with("/shared-services/certs", True) + self.assertEqual(len(self.lookups), 1) + + async def test_existing_nexus_prevents_recovery_and_creation(self): + self.find_service.return_value = {"id": "existing-nexus"} + with self.assertRaisesRegex(pytest.fail.Exception, "Nexus is present"): + await shared.test_create_certificate_shared_service(True) + self.recover.assert_not_awaited() + self.post.assert_not_awaited() + self.cleanup.assert_not_awaited() + + async def test_weekend_skip_does_not_access_azure(self): + for day in (5, 6): + with self.subTest(day=day): + self.date.today.return_value.weekday.return_value = day + with self.assertRaisesRegex(pytest.skip.Exception, "coverage is unproven"): + await shared.test_create_certificate_shared_service(True) + self.token.assert_not_awaited() + self.recover.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_explicit_weekend_override_allows_the_certificate_case(self): + self.date.today.return_value.weekday.return_value = 5 + with patch.object(shared.config, "TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS", True): + await shared.test_create_certificate_shared_service(True) + self.post.assert_awaited_once() + self.cleanup.assert_awaited_once() + + async def test_wrong_deployed_record_fails_and_still_cleans_up(self): + self.record["deploymentStatus"] = "deployment_failed" + with self.assertRaises(AssertionError): + await shared.test_create_certificate_shared_service(True) + self.cleanup.assert_awaited_once_with("/shared-services/certs", True) + + async def test_body_error_survives_a_cleanup_failure(self): + failure = RuntimeError("Record lookup failed") + self.get.side_effect = failure + self.cleanup.side_effect = RuntimeError("Delete failed") + with self.assertRaises(RuntimeError) as caught: + await shared.test_create_certificate_shared_service(True) + self.assertIs(caught.exception, failure) + self.assertIn("Delete failed", failure.__notes__[0]) + + async def test_nexus_appearing_during_the_case_retains_the_certificate(self): + self.find_service.side_effect = [None, {"id": "new-nexus"}] + with self.assertRaisesRegex(RuntimeError, "Retaining certificate dependency"): + await shared.test_create_certificate_shared_service(True) + self.cleanup.assert_not_awaited() + self.assertEqual(self.lookups, []) + + async def test_visible_record_after_deletion_fails_the_case(self): + for code in (200, 403, 500): + with self.subTest(code=code): + self.lookup_status = code + with self.assertRaisesRegex(AssertionError, f"HTTP {code}"): + await shared.test_create_certificate_shared_service(True) + + async def test_cleanup_uses_its_own_budget_after_validation_timeout(self): + async def lookup(*args): + await asyncio.sleep(60) + + async def cleanup(*args): + await asyncio.sleep(0.03) + + self.get.side_effect = lookup + self.cleanup.side_effect = cleanup + with patch.object(shared, "CERTIFICATE_CHECK_TIMEOUT_SECONDS", 0.01): + with self.assertRaises(TimeoutError): + await shared.test_create_certificate_shared_service(True) + self.cleanup.assert_awaited_once_with("/shared-services/certs", True) + + async def test_cancellation_still_cleans_up_the_owned_certificate(self): + self.get.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await shared.test_create_certificate_shared_service(True) + self.cleanup.assert_awaited_once_with("/shared-services/certs", True) + + async def test_creation_error_is_preserved_for_existing_failed_create_recovery(self): + failure = RuntimeError("Certificate install failed") + self.post.side_effect = failure + with self.assertRaises(RuntimeError) as caught: + await shared.test_create_certificate_shared_service(True) + self.assertIs(caught.exception, failure) + self.assertTrue(self.post.await_args.kwargs["cleanup_failed_create"]) + self.cleanup.assert_not_awaited() + + async def test_recovery_timeout_prevents_new_creation(self): + async def recover(*args): + await asyncio.sleep(60) + + self.recover.side_effect = recover + with patch.object(shared, "RECOVERY_TIMEOUT_SECONDS", 0.01): + with self.assertRaises(TimeoutError): + await shared.test_create_certificate_shared_service(True) + self.post.assert_not_awaited() diff --git a/e2e_tests/unit_tests/test_reused_prerequisite_evidence.py b/e2e_tests/unit_tests/test_reused_prerequisite_evidence.py new file mode 100644 index 0000000000..eb2c0d5309 --- /dev/null +++ b/e2e_tests/unit_tests/test_reused_prerequisite_evidence.py @@ -0,0 +1,76 @@ +"""Record identities from actual prerequisite selection without resource writes.""" + +from contextlib import asynccontextmanager +from pathlib import Path +import tempfile +import unittest +from unittest.mock import AsyncMock, patch + +from e2e_tests import bundle_evidence +from e2e_tests.resources import strings + +try: + from e2e_tests.resources import nexus as prerequisites + + nexus_context = prerequisites.nexus_prerequisites +except ImportError: + from e2e_tests import test_guacamole_linuxvm as prerequisites + + nexus_context = prerequisites.linux_vm_nexus + + +class ReusedPrerequisiteEvidenceTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.directory = self.enterContext(tempfile.TemporaryDirectory()) + self.report = bundle_evidence.BundleReport(Path(self.directory) / "report.json", {}) + bundle_evidence.activate(self.report) + self.addCleanup(bundle_evidence.activate, None) + self.enterContext(patch.object(prerequisites, "get_admin_token", AsyncMock(return_value="secret-token"))) + self.certs = { + "id": "cert-id", + "templateName": strings.CERTS_SHARED_SERVICE, + "templateVersion": "1.2.3", + "deploymentStatus": "deployed", + "isEnabled": True, + "properties": {"domain_prefix": "nexus", "cert_name": "cert-name", "secret": "never-copy"}, + } + self.nexus = { + "id": "nexus-id", + "templateName": strings.NEXUS_SHARED_SERVICE, + "templateVersion": "3.11.0", + "deploymentStatus": "deployed", + "isEnabled": True, + "properties": {"ssl_cert_name": "cert-name", "secret": "never-copy"}, + } + + async def test_reused_nexus_and_matching_certificate_are_recorded_without_writes(self): + unrelated = {**self.certs, "id": "unrelated", "properties": {"cert_name": "another-cert"}} + with ( + patch.object( + prerequisites, + "get_resource", + AsyncMock(return_value={"sharedServices": [self.nexus, self.certs, unrelated]}), + ), + patch.object(prerequisites, "temporary_resource", side_effect=AssertionError("Unexpected write")), + ): + async with nexus_context(True): + pass + self.assertEqual({r["id"] for r in self.report.data["reused_resources"]}, {"nexus-id", "cert-id"}) + self.assertEqual(self.report.data["reused_resources"][0]["templateVersion"], "3.11.0") + self.assertEqual(self.report.data["resources"], []) + self.assertNotIn("never-copy", self.report.filename.read_text()) + self.assertNotIn("secret-token", self.report.filename.read_text()) + + async def test_existing_certificate_is_recorded_when_new_nexus_is_created(self): + @asynccontextmanager + async def create(*args, **kwargs): + yield "/shared-services/new-nexus" + + with ( + patch.object(prerequisites, "get_resource", AsyncMock(return_value={"sharedServices": [self.certs]})), + patch.object(prerequisites, "temporary_resource", create), + patch.object(prerequisites.config, "TEST_ACCEPT_NEXUS_EULA", True), + ): + async with nexus_context(True): + pass + self.assertEqual([r["id"] for r in self.report.data["reused_resources"]], ["cert-id"]) From 34f29eb69e84c437ecbeaabcc450fa41bdd9aa4f Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:42:42 +0100 Subject: [PATCH 07/17] Validate base workspace and Azure SQL lifecycles (#5119) (cherry picked from commit cb06eb9c689f29582a740f01cea13914822f32a3) --- .github/workflows/deploy_tre_reusable.yml | 5 + CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 40 ++- docs/tre-developers/github-pr-bot-commands.md | 2 +- e2e_tests/bundle_coverage.json | 32 +- e2e_tests/pytest.ini | 2 + e2e_tests/resources/deployment.py | 47 ++- e2e_tests/resources/sql_lifecycle.py | 146 +++++++++ e2e_tests/resources/sql_probe.ps1 | 74 +++++ e2e_tests/resources/sql_probe.py | 210 +++++++++++++ e2e_tests/test_azuresql.py | 90 ++++++ e2e_tests/test_workspace_base.py | 82 +++++ .../test_base_workspace_lifecycle.py | 234 ++++++++++++++ e2e_tests/unit_tests/test_bundle_selection.py | 18 +- .../unit_tests/test_polling_diagnostics.py | 132 ++++++++ e2e_tests/unit_tests/test_sql_deadlines.py | 228 ++++++++++++++ e2e_tests/unit_tests/test_sql_lifecycle.py | 126 ++++++++ e2e_tests/unit_tests/test_sql_probe.py | 293 ++++++++++++++++++ 18 files changed, 1749 insertions(+), 13 deletions(-) create mode 100644 e2e_tests/resources/sql_lifecycle.py create mode 100644 e2e_tests/resources/sql_probe.ps1 create mode 100644 e2e_tests/resources/sql_probe.py create mode 100644 e2e_tests/test_azuresql.py create mode 100644 e2e_tests/test_workspace_base.py create mode 100644 e2e_tests/unit_tests/test_base_workspace_lifecycle.py create mode 100644 e2e_tests/unit_tests/test_polling_diagnostics.py create mode 100644 e2e_tests/unit_tests/test_sql_deadlines.py create mode 100644 e2e_tests/unit_tests/test_sql_lifecycle.py create mode 100644 e2e_tests/unit_tests/test_sql_probe.py diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index 3ea66c973f..a8dcf562bb 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -1064,6 +1064,10 @@ jobs: [deploy_shared_services, register_bundles, register_user_resource_bundles, e2e_tests_smoke] timeout-minutes: 300 steps: + - name: Reserve SQL cleanup and reporting time + if: inputs.e2eBundle == 'tre-workspace-service-azuresql' + run: echo "SQL_VALIDATION_DEADLINE=$(( $(date +%s) + 270 * 60 ))" >> "$GITHUB_ENV" + - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -1090,6 +1094,7 @@ jobs: with: COMMAND: >- TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS=${{ inputs.runCertificateTestsOnWeekends }} + SQL_VALIDATION_DEADLINE=${{ env.SQL_VALIDATION_DEADLINE }} TEST_BUNDLE_REQUEST_FILE=e2e_tests/bundle-validation-request.json make test-e2e-bundle DEVCONTAINER_TAG: ${{ inputs.DEVCONTAINER_TAG }} CI_CACHE_ACR_NAME: ${{ secrets.CI_CACHE_ACR_NAME}} diff --git a/CHANGELOG.md b/CHANGELOG.md index b8da604d70..b5ffb4133a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: +* Validate base-workspace and private Azure SQL lifecycles with bounded cleanup and safe operation-polling diagnostics. ([#5257](https://github.com/microsoft/AzureTRE/pull/5257)) * Select individual bundle tests, report coverage limits and run selected tests after successful smoke tests. ([#5256](https://github.com/microsoft/AzureTRE/pull/5256)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index bdb62a9cc7..528c251801 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -25,6 +25,17 @@ Use the "Run and Debug" panel within Visual Studio Code, select "E2E Extended", - This will also use `/workspaces/AzureTRE/core/private.env` file for other values. +## Investigate operation-polling failures + +Unexpected HTTP responses fail operation polling. HTTP 500 is not retried or treated as successful deletion. +The error log records a UTC timestamp, method, recognised operation path, status, media type, body size and coarse body classification. +It includes allowlisted correlation headers only when their values match supported identifier formats. +Unrecognised paths and header values are omitted or redacted. The diagnostic record excludes response bodies, credentials, host names and URL parameters. + +If a poll fails, use its timestamp, operation ID and available correlation IDs to find the API exception and App Service request record. +Check API exception telemetry before a validation run. Record terminal resource deletion separately from the failed poll. +These diagnostics help investigate the failure. They do not establish its cause or guarantee that server telemetry is available. + ## Validate one bundle The [bundle coverage map](https://github.com/microsoft/AzureTRE/blob/main/e2e_tests/bundle_coverage.json) lists every Porter bundle, its prerequisite bundles, exact pytest cases and remaining coverage gaps. @@ -39,7 +50,9 @@ Use the E2E Python environment from the repository root: python3 e2e_tests/run_bundle.py --bundle tre-workspace-service-azuresql --collect-only ``` -This collects the named Azure SQL parametrisation without creating resources. +This collects `test_azuresql.py::test_sql_query_survives_sku_upgrade` without creating resources. +The case checks a private SQL query and data persistence after an S1-to-S2 SKU change. + The report records `not_run`, even when collection succeeds. Unknown names, unavailable cases, missing tests and unexpected collected cases fail explicitly. @@ -76,6 +89,31 @@ It is excluded from the ordinary `shared_services` selection to avoid an additio The existing combined case remains in that suite. It deletes Nexus before its certificate dependency. Certificate content, renewal, retained Key Vault artefacts, Nexus package/proxy use, upgrades and physical resource removal need separate evidence. +### Base workspace lifecycle + +Select `tre-workspace-base` to run `test_workspace_base.py::test_base_workspace_lifecycle` independently. +The case always creates its own workspace with Automatic authentication and backups disabled. It does not reuse `TEST_WORKSPACE_ID`. +It checks the deployed record and uses the configured workspace identity to list the new workspace's services. +It then disables and deletes the workspace, waits for the delete operation and confirms that an API lookup returns HTTP 404. +Body failures still trigger cleanup. If cleanup also fails, the original failure is preserved with an additional diagnostic note. + +Setup uses the existing one-hour deployment limit and failed-create recovery. The API checks have a five-minute limit. +Cleanup has a separate one-hour budget. The case is opt-in through bundle selection or the `workspace_validation` marker. +It does not add another workspace to the extended suite. + +These checks do not prove network isolation, storage access, upgrades, custom actions or backup restore and retention. +Verify Azure resource removal separately when recording release evidence. + +### SQL time limits + +The SQL bundle job has a 300-minute limit. Its first step sets an absolute lifecycle deadline at 270 minutes. +Setup and queries stop with 120 minutes remaining for cleanup. Container startup uses the same budget. +The SQL case manages its own workspace and Guacamole service so that setup and cleanup share this deadline. +It preserves pre-created resources and divides the remaining cleanup time between the resources it created. +Failed-create recovery and temporary probe cleanup also count towards the lifecycle deadline. +The final 30 minutes are reserved for results and job overhead. Cleanup failures still fail the test. +For a local run, the 270-minute lifecycle budget starts when the SQL case begins. + ### Run the branch workflow before merge 1. Push the reviewed changes to a branch in `microsoft/AzureTRE`. diff --git a/docs/tre-developers/github-pr-bot-commands.md b/docs/tre-developers/github-pr-bot-commands.md index edda0af0cc..3d08cf5093 100644 --- a/docs/tre-developers/github-pr-bot-commands.md +++ b/docs/tre-developers/github-pr-bot-commands.md @@ -22,7 +22,7 @@ The table describes the selected assertions. A passing command does not prove ev | `/test-airlock` | Import requests, access checks, the import review VM flow and export request storage routing | No export review VM or migration proof. | | `/test-backups` | Base workspace deployment with backups enabled and disabled | No backup restore or retention proof. | -The `workspace_services`, `linux_vm` and `airlock_validation` markers have no dedicated slash command. +The `workspace_services`, `linux_vm`, `airlock_validation` and `sql_validation` markers have no dedicated slash command. Use the branch workflow or the local test runner for these selections. The `airlock_validation` marker selects two Airlock cases. It does not select the full `airlock` suite. See [bundle selection](end-to-end-tests.md#validate-one-bundle) for exact cases and declared coverage gaps. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 32a009dae0..f3a1788d49 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -183,18 +183,25 @@ "manifest": "templates/workspace_services/azuresql/porter.yaml", "family": "workspace_service", "prerequisites": [ - "tre-workspace-base" + "tre-workspace-base", + "tre-service-guacamole", + "tre-service-guacamole-windowsvm" ], "tests": [ + "test_azuresql.py::test_sql_query_survives_sku_upgrade" + ], + "related_tests": [ "test_workspace_services.py::test_install_workspace_service[tre-workspace-service-azuresql]" ], - "related_tests": [], "coverage": [ "install", + "private SQL write/read", + "S1-to-S2 configuration upgrade with retained data", "uninstall" ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "AzureCloud only. Requires permission to run commands on a test-created workspace VM and create a temporary secret-scoped role assignment.", + "Configuration upgrade uses the current bundle version. Previous-version migration remains separate (#5128). The SQL bundle has no custom actions." ], "unavailable_reason": "" }, @@ -430,15 +437,26 @@ "manifest": "templates/workspaces/base/porter.yaml", "family": "workspace", "prerequisites": [], - "tests": [], + "tests": [ + "test_workspace_base.py::test_base_workspace_lifecycle" + ], "related_tests": [ "test_workspace_services.py::test_create_guacamole_service_into_base_workspace" ], - "coverage": [], + "coverage": [ + "install", + "deployment-status", + "workspace-identity-api-access", + "uninstall", + "api-removal" + ], "limitations": [ - "Functional, upgrade and custom-action coverage must be assessed separately." + "Creates a new Automatic-auth workspace even when TEST_WORKSPACE_ID is configured. Backups are disabled.", + "Workspace service-list access checks the configured workspace identity, not network isolation or storage access.", + "The case checks terminal uninstall and an API 404. Independent Azure resource inventory remains a separate check.", + "Upgrade, custom-action, backup configuration, restore and retention coverage must be assessed separately." ], - "unavailable_reason": "Base workspace provisioning is fixture coverage in other bundle tests; an isolated lifecycle case is missing (#5152)." + "unavailable_reason": "" }, "tre-workspace-unrestricted": { "manifest": "templates/workspaces/unrestricted/porter.yaml", diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index 866dcfdffc..b810211fe8 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -14,6 +14,8 @@ markers = airlock: only airlock related airlock_validation: draft-container deletion and review-VM flow, including Nexus prerequisites and cleanup workspace_services + workspace_validation: opt-in base-workspace creation, scoped API access and deletion + sql_validation: Private SQL query and configuration upgrade lifecycle asyncio_mode = auto asyncio_default_fixture_loop_scope = session diff --git a/e2e_tests/resources/deployment.py b/e2e_tests/resources/deployment.py index 8d5ffb0e8b..6189d6374e 100644 --- a/e2e_tests/resources/deployment.py +++ b/e2e_tests/resources/deployment.py @@ -1,10 +1,54 @@ import logging +import re +from datetime import datetime, timezone import backoff from httpx import TimeoutException from e2e_tests.helpers import get_auth_header, get_full_endpoint from e2e_tests.resources import strings LOGGER = logging.getLogger(__name__) +_UUID = r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +_OPERATION_PATH = re.compile( + rf"/api/(?:workspaces/{_UUID}/(?:workspace-services/{_UUID}/(?:user-resources/{_UUID}/)?)?" + rf"|shared-services/{_UUID}/)?operations/{_UUID}" +) +_CORRELATION_HEADERS = { + "x-ms-request-id": _UUID, + "x-ms-correlation-request-id": _UUID, + "x-request-id": _UUID, + "request-id": rf"(?:{_UUID}|\|[0-9a-f]{32}\.[0-9a-f]{16}\.)", + "traceparent": r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", +} + + +def polling_failure_details(response, operation_endpoint): + """Return bounded metadata. Never copy bodies, URL parameters or arbitrary headers.""" + path = operation_endpoint.split("?", 1)[0].split("#", 1)[0] + media_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower() + allowed_media_types = {"application/json", "application/problem+json", "text/plain", "text/html"} + prefix = response.content[:64].lstrip().lower() + if not response.content: + body_kind = "empty" + elif prefix.startswith((b"{", b"[")): + body_kind = "json_like" + elif prefix.startswith((b"", + "status": response.status_code, + "content_type": media_type if media_type in allowed_media_types else "other_or_missing", + "body_bytes": len(response.content), + "body_kind": body_kind, + "correlation": { + name: value + for name, pattern in _CORRELATION_HEADERS.items() + if (value := response.headers.get(name, "")) and re.fullmatch(pattern, value) + }, + } async def delete_done(client, operation_endpoint, access_token): @@ -58,8 +102,7 @@ async def check_deployment(client, operation_endpoint, access_token): LOGGER.info(f"Resource not found (404) - treating as deleted: {operation_endpoint}") return strings.RESOURCE_STATUS_DELETED, "Resource has been deleted", "" else: - LOGGER.error(f"Non 200 response in check_deployment: {response.status_code}") - LOGGER.error(f"Full response: {response}") + LOGGER.error("Operation polling failed: %s", polling_failure_details(response, operation_endpoint)) raise Exception("Non 200 response in check_deployment") diff --git a/e2e_tests/resources/sql_lifecycle.py b/e2e_tests/resources/sql_lifecycle.py new file mode 100644 index 0000000000..b574849fca --- /dev/null +++ b/e2e_tests/resources/sql_lifecycle.py @@ -0,0 +1,146 @@ +"""Reserve cleanup time for all resources created by the SQL validation case.""" + +import asyncio +from builtins import ExceptionGroup +from contextlib import asynccontextmanager +from functools import partial +import logging +import math +import os +import time + +from e2e_tests import config +from e2e_tests.conftest import ( + clean_up_test_workspace, + clean_up_test_workspace_service, + create_or_get_test_workspace, + create_or_get_test_workpace_service, + get_workspace_owner_token, +) +from e2e_tests.resources.resource import disable_and_delete_resource, get_resource, post_resource +from e2e_tests.timeouts import cleanup_deadline + +LOGGER = logging.getLogger(__name__) +LIFECYCLE_SECONDS = 270 * 60 +CLEANUP_RESERVE_SECONDS = 120 * 60 +TERMINAL_STATES = { + "deployed", + "deployment_failed", + "updated", + "updating_failed", + "deleted", + "deleting_failed", + "action_succeeded", + "action_failed", + "pipeline_succeeded", + "pipeline_failed", +} + + +def lifecycle_deadlines(): + """Use the job's absolute deadline, including time spent starting the container.""" + loop = asyncio.get_running_loop() + configured = os.environ.get("SQL_VALIDATION_DEADLINE", "") + deadline = float(configured) if configured else time.time() + LIFECYCLE_SECONDS + if not math.isfinite(deadline): + raise ValueError("SQL validation deadline must be finite") + remaining = min(LIFECYCLE_SECONDS, deadline - time.time()) + if not 0 < remaining <= LIFECYCLE_SECONDS or remaining <= CLEANUP_RESERVE_SECONDS: + raise TimeoutError("Insufficient SQL validation time remains after reserving cleanup") + finish = loop.time() + remaining + return finish - CLEANUP_RESERVE_SECONDS, finish + + +class SQLResources: + def __init__(self, verify, finish): + self.verify = verify + self.finish = finish + self.finalisers = [] + + def own(self, name, callback, *args, **kwargs): + self.finalisers.append((name, partial(callback, *args, **kwargs))) + + async def setup(self): + # Keep these resources inside the SQL budget rather than session fixtures. + workspace_path, workspace_id = await create_or_get_test_workspace( + auth_type="Manual" if config.TEST_WORKSPACE_APP_ID else "Automatic", + verify=self.verify, + pre_created_workspace_id=config.TEST_WORKSPACE_ID, + client_id=config.TEST_WORKSPACE_APP_ID, + client_secret=config.TEST_WORKSPACE_APP_SECRET, + ) + if not config.TEST_WORKSPACE_ID: + self.own(workspace_path, clean_up_test_workspace, "", workspace_path, self.verify) + token = await get_workspace_owner_token(workspace_id, self.verify) + guacamole_path, guacamole_id = await create_or_get_test_workpace_service( + workspace_path, token, config.TEST_WORKSPACE_SERVICE_ID, self.verify + ) + if not config.TEST_WORKSPACE_SERVICE_ID: + self.own(guacamole_path, clean_up_test_workspace_service, "", guacamole_path, workspace_id, self.verify) + return workspace_path, workspace_id, guacamole_path, guacamole_id + + async def create(self, payload, endpoint, token): + async with asyncio.timeout(60 * 60): + resource_path, resource_id = await post_resource( + payload, endpoint, token, self.verify, cleanup_failed_create=True + ) + self.own(resource_path, self.delete, resource_path, token) + return resource_path, resource_id + + async def delete(self, resource_path, token): + while True: + operations = (await get_resource(f"/api{resource_path}/operations", token, self.verify))["operations"] + if all(op["status"] in TERMINAL_STATES for op in operations): + break + await asyncio.sleep(30) + await disable_and_delete_resource(f"/api{resource_path}", token, self.verify, allow_failed_disable=True) + + async def close(self): + failures = [] + while self.finalisers: + # Each remaining resource gets a share. One slow delete must not use + # the entire reserve before parent cleanup can even start. + seconds = (self.finish - asyncio.get_running_loop().time()) / len(self.finalisers) + name, callback = self.finalisers.pop() + try: + async with cleanup_deadline(seconds): + await callback() + except Exception as error: + error.add_note(f"SQL validation cleanup failed for {name}") + LOGGER.error("SQL validation cleanup failed for %s: %r", name, error) + failures.append(error) + return failures + + +@asynccontextmanager +async def sql_lifecycle(verify): + work_finish, finish = lifecycle_deadlines() + resources = SQLResources(verify, finish) + original = None + # Accepted-create recovery inherits the same absolute limit. It cannot add + # another hour after the lifecycle deadline. + async with cleanup_deadline(finish - asyncio.get_running_loop().time()): + try: + async with asyncio.timeout_at(work_finish): + yield resources + except BaseException as error: + original = error + raise + finally: + # Repeated caller cancellation must wait for bounded recovery. + cleanup = asyncio.create_task(resources.close()) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = error + failures = cleanup.result() + failure = original or cancelled + if failure is not None: + for error in failures: + failure.add_note(f"SQL resource cleanup also failed: {error!r}; {error.__notes__}") + elif failures: + raise ExceptionGroup("SQL resource cleanup failed", failures) + if original is None and cancelled is not None: + raise cancelled diff --git a/e2e_tests/resources/sql_probe.ps1 b/e2e_tests/resources/sql_probe.ps1 new file mode 100644 index 0000000000..6075089639 --- /dev/null +++ b/e2e_tests/resources/sql_probe.ps1 @@ -0,0 +1,74 @@ +param( + [Parameter(Mandatory)][ValidatePattern('^[a-z0-9-]+\.database\.windows\.net$')][string]$Server, + [Parameter(Mandatory)][ValidatePattern('^[a-z0-9-]+$')][string]$VaultName, + [Parameter(Mandatory)][ValidatePattern('^[a-z0-9-]+$')][string]$SecretName, + [Parameter(Mandatory)][ValidatePattern('^[a-f0-9]{32}$')][string]$ProbeId, + [Parameter(Mandatory)][ValidateSet('write', 'read')][string]$Phase, + [Parameter(Mandatory)][ValidateSet('S1', 'S2')][string]$ExpectedSku +) +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$connection = $null +$stage = 'private-dns' +try { + $addresses = [System.Net.Dns]::GetHostAddresses($Server) + if ($addresses.Count -eq 0) { throw 'No SQL address' } + foreach ($address in $addresses) { + $bytes = $address.GetAddressBytes() + $private = $bytes.Length -eq 4 -and ( + $bytes[0] -eq 10 -or + ($bytes[0] -eq 172 -and $bytes[1] -ge 16 -and $bytes[1] -le 31) -or + ($bytes[0] -eq 192 -and $bytes[1] -eq 168) + ) + if (-not $private) { throw 'SQL DNS is not private' } + } + $stage = 'secret-access' + $secret = $null + $deadline = [DateTime]::UtcNow.AddMinutes(5) + do { + try { + $identity = Invoke-RestMethod -Headers @{Metadata = 'true'} -Uri 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net' -TimeoutSec 15 + $secret = Invoke-RestMethod -Headers @{Authorization = "Bearer $($identity.access_token)"} -Uri "https://$VaultName.vault.azure.net/secrets/${SecretName}?api-version=7.4" -TimeoutSec 15 + } catch { + if ([DateTime]::UtcNow -ge $deadline) { throw 'Secret access did not become ready' } + Start-Sleep -Seconds 10 + } + } while ($null -eq $secret) + $stage = 'sql-connect' + $builder = New-Object System.Data.SqlClient.SqlConnectionStringBuilder + $builder['Data Source'] = "tcp:$Server,1433" + $builder['Initial Catalog'] = 'tredb' + $builder['User ID'] = 'azuresqladmin' + $builder['Password'] = $secret.value + $builder['Encrypt'] = $true + $builder['TrustServerCertificate'] = $false + $builder['Connect Timeout'] = 30 + $connection = New-Object System.Data.SqlClient.SqlConnection($builder.ConnectionString) + $connection.Open() + $stage = 'sql-data' + $command = $connection.CreateCommand() + $command.CommandTimeout = 60 + $null = $command.Parameters.Add('@probe', [System.Data.SqlDbType]::VarChar, 32) + $command.Parameters['@probe'].Value = $ProbeId + if ($Phase -eq 'write') { + $command.CommandText = 'CREATE TABLE dbo.tre_e2e_probe (probe varchar(32) NOT NULL PRIMARY KEY); INSERT INTO dbo.tre_e2e_probe (probe) VALUES (@probe);' + $null = $command.ExecuteNonQuery() + } + $command.CommandText = 'SELECT COUNT(*) FROM dbo.tre_e2e_probe WHERE probe = @probe' + $count = [int]$command.ExecuteScalar() + if ($count -ne 1) { throw 'Expected SQL row is missing' } + $command.CommandText = "SELECT CAST(DATABASEPROPERTYEX(DB_NAME(), 'ServiceObjective') AS nvarchar(128))" + $sku = [string]$command.ExecuteScalar() + if ($sku -ne $ExpectedSku) { throw 'Unexpected SQL SKU' } + $result = @{phase = $Phase; probe = $ProbeId; sku = $sku; row_count = $count; private_dns = $true} + Write-Output ('SQL_PROBE_RESULT=' + ($result | ConvertTo-Json -Compress)) +} catch { + # Do not emit exception details, tokens, connection strings or secret values. + Write-Output "SQL_PROBE_FAILED=$stage" + exit 1 +} finally { + if ($null -ne $connection) { $connection.Dispose() } + $secret = $null + $identity = $null + $builder = $null +} diff --git a/e2e_tests/resources/sql_probe.py b/e2e_tests/resources/sql_probe.py new file mode 100644 index 0000000000..e72333139f --- /dev/null +++ b/e2e_tests/resources/sql_probe.py @@ -0,0 +1,210 @@ +"""Run a private SQL probe from a test-created workspace VM.""" + +import asyncio +from contextlib import asynccontextmanager +import json +import os +from pathlib import Path +import re +from urllib.parse import urlparse +from uuid import uuid4 + +from azure.identity import AzureCliCredential, ClientAssertionCredential +from httpx import AsyncClient +import requests + +from e2e_tests.timeouts import cleanup_deadline + + +ARM = "https://management.azure.com" +SECRETS_USER = "4633458b-17de-408a-b874-0445c86b69e6" +COMPUTE_API = "2023-03-01" +ROLE_API = "2022-04-01" +SCRIPT = Path(__file__).with_name("sql_probe.ps1") + + +class ArmClient: + """Use CI's existing CLI login without exposing response bodies on errors.""" + + def __init__(self, client, credential): + self.client = client + self.credential = credential + + async def request(self, method, resource_id, api_version, *, body=None, missing_ok=False, expand=False): + if not resource_id.startswith("/subscriptions/") or any(c in resource_id for c in "?#"): + raise ValueError("Expected an ARM resource ID") + token = await asyncio.to_thread(self.credential.get_token, f"{ARM}/.default") + params = {"api-version": api_version} + if expand: + params["$expand"] = "instanceView" + response = await self.client.request( + method, + f"{ARM}{resource_id}", + params=params, + headers={"Authorization": f"Bearer {token.token}"}, + json=body, + ) + if missing_ok and response.status_code == 404: + return None + if response.status_code not in (200, 201, 202, 204): + raise RuntimeError(f"ARM {method} {resource_id} failed with HTTP {response.status_code}") + return response.json() if response.content else {} + + async def delete(self, resource_id, api_version): + async with asyncio.timeout(180): + await self.request("DELETE", resource_id, api_version, missing_ok=True) + while await self.request("GET", resource_id, api_version, missing_ok=True) is not None: + await asyncio.sleep(5) + + +@asynccontextmanager +async def arm_client(): + # Long deployments can outlive azure/login's cached token. Request a fresh + # GitHub assertion when Azure Identity needs to renew the ARM token. + if os.environ.get("ACTIONS_ID_TOKEN_REQUEST_URL"): + credential = ClientAssertionCredential( + os.environ["ARM_TENANT_ID"], os.environ["ARM_CLIENT_ID"], github_assertion + ) + else: + credential = AzureCliCredential() + with credential: + async with AsyncClient(timeout=60) as client: + yield ArmClient(client, credential) + + +def github_assertion(): + url = os.environ["ACTIONS_ID_TOKEN_REQUEST_URL"] + parsed = urlparse(url) + if parsed.scheme != "https" or not (parsed.hostname or "").endswith(".actions.githubusercontent.com"): + raise ValueError("Unexpected GitHub OIDC endpoint") + response = requests.get( + url, + params={"audience": "api://AzureADTokenExchange"}, + headers={"Authorization": f"Bearer {os.environ['ACTIONS_ID_TOKEN_REQUEST_TOKEN']}"}, + timeout=30, + allow_redirects=False, + ) + if response.status_code != 200: + raise RuntimeError(f"GitHub OIDC request failed with HTTP {response.status_code}") + return response.json()["value"] + + +def validate_vm(vm, vm_id, subscription_id, tre_id, workspace_id, service_id, resource_id): + expected = { + "tre_id": tre_id, + "tre_workspace_id": workspace_id, + "tre_workspace_service_id": service_id, + "tre_user_resource_id": resource_id, + } + group = f"/subscriptions/{subscription_id}/resourceGroups/rg-{tre_id}-ws-{workspace_id[-4:]}" + if not re.fullmatch(re.escape(group) + r"/providers/Microsoft.Compute/virtualMachines/[a-zA-Z0-9-]+", vm_id, re.I): + raise ValueError("SQL probe VM is outside the expected workspace") + if vm.get("id", "").lower() != vm_id.lower() or any(vm.get("tags", {}).get(k) != v for k, v in expected.items()): + raise ValueError("SQL probe VM does not match the test-created resource") + if not vm.get("identity", {}).get("principalId"): + raise ValueError("SQL probe VM has no system-assigned identity") + return group + + +@asynccontextmanager +async def delete_after(arm, resource_id, api_version): + """Finish bounded probe cleanup before propagating caller cancellation.""" + original = None + try: + yield + except BaseException as error: + original = error + raise + finally: + + async def remove(): + async with cleanup_deadline(180): + await arm.delete(resource_id, api_version) + + cleanup = asyncio.create_task(remove()) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = error + except Exception: + break + try: + cleanup.result() + except (Exception, asyncio.CancelledError) as error: + failure = original or cancelled + if failure is None: + raise + failure.add_note(f"Cleanup of {resource_id} also failed: {error!r}") + if original is None and cancelled is not None: + raise cancelled + + +@asynccontextmanager +async def secret_access(arm, vault_id, secret_name, principal_id, subscription_id): + """Grant only this VM access to this SQL secret, then verify revocation.""" + assignment = f"{vault_id}/secrets/{secret_name}/providers/Microsoft.Authorization/roleAssignments/{uuid4()}" + body = { + "properties": { + "roleDefinitionId": f"/subscriptions/{subscription_id}/providers/Microsoft.Authorization/roleDefinitions/{SECRETS_USER}", + "principalId": principal_id, + "principalType": "ServicePrincipal", + } + } + # The PUT may have succeeded even if its response was lost. + async with delete_after(arm, assignment, ROLE_API): + await arm.request("PUT", assignment, ROLE_API, body=body) + yield + + +def probe_result(resource, phase, probe_id, expected_sku): + properties = resource.get("properties", {}) + view = properties.get("instanceView", {}) + state = view.get("executionState", "") + if properties.get("provisioningState") in ("Failed", "Canceled") or state in ("Failed", "Canceled", "TimedOut"): + raise AssertionError(f"SQL probe failed: state={state}, exitCode={view.get('exitCode')}") + if state != "Succeeded": + return False + if view.get("exitCode") != 0: + raise AssertionError("SQL probe returned a non-zero exit code") + marker = "SQL_PROBE_RESULT=" + results = [line[len(marker) :] for line in view.get("output", "").splitlines() if line.startswith(marker)] + expected = {"phase": phase, "probe": probe_id, "sku": expected_sku, "row_count": 1, "private_dns": True} + if len(results) != 1 or json.loads(results[0]) != expected: + raise AssertionError("SQL probe did not confirm the expected data, SKU and private DNS") + return True + + +async def run_probe(arm, vm_id, location, *, server, vault_name, secret_name, probe_id, phase, expected_sku): + parameters = { + "Server": server, + "VaultName": vault_name, + "SecretName": secret_name, + "ProbeId": probe_id, + "Phase": phase, + "ExpectedSku": expected_sku, + } + command = f"{vm_id}/runCommands/sql-probe-{uuid4().hex}" + # Removing our command also terminates it if its deadline expired. + async with delete_after(arm, command, COMPUTE_API): + async with asyncio.timeout(15 * 60): + await arm.request( + "PUT", + command, + COMPUTE_API, + body={ + "location": location, + "properties": { + "source": {"script": SCRIPT.read_text()}, + "parameters": [{"name": k, "value": v} for k, v in parameters.items()], + "timeoutInSeconds": 600, + "treatFailureAsDeploymentFailure": True, + }, + }, + ) + while True: + result = await arm.request("GET", command, COMPUTE_API, expand=True) + if probe_result(result, phase, probe_id, expected_sku): + return + await asyncio.sleep(10) diff --git a/e2e_tests/test_azuresql.py b/e2e_tests/test_azuresql.py new file mode 100644 index 0000000000..25d09d04b5 --- /dev/null +++ b/e2e_tests/test_azuresql.py @@ -0,0 +1,90 @@ +"""Exercise SQL data access and a supported configuration upgrade privately.""" + +import asyncio +import logging +import os +from uuid import uuid4 + +import pytest + +from e2e_tests import config +from e2e_tests.conftest import get_workspace_owner_token +from e2e_tests.resources import strings +from e2e_tests.resources.resource import get_resource, post_resource +from e2e_tests.resources.sql_lifecycle import sql_lifecycle +from e2e_tests.resources.sql_probe import arm_client, COMPUTE_API, run_probe, secret_access, validate_vm + +LOGGER = logging.getLogger(__name__) +SQL_PAYLOAD = { + "templateName": strings.AZURESQL_SERVICE, + "properties": { + "display_name": "SQL lifecycle test", + "description": "Private SQL query and SKU upgrade validation", + "sql_sku": "S1 | 20 DTUs", + "db_name": "tredb", + }, +} +VM_PAYLOAD = { + "templateName": strings.GUACAMOLE_WINDOWS_USER_RESOURCE, + "properties": { + "display_name": "SQL test client", + "description": "Temporary private SQL validation client", + "os_image": "Windows 11", + "vm_size": "2 CPU | 8GB RAM", + "admin_username": "researcher", + "shared_storage_access": False, + **{ + f"install_{tool}": False + for tool in ("azure_cli", "vscode", "storage_explorer", "git", "python_tools", "r_tools") + }, + }, +} + + +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.sql_validation +@pytest.mark.timeout(285 * 60) +async def test_sql_query_survives_sku_upgrade(verify): + if os.environ.get("AZURE_ENVIRONMENT", "AzureCloud") != "AzureCloud": + raise ValueError("The SQL probe currently supports AzureCloud only") + async with sql_lifecycle(verify) as resources: + workspace_path, workspace_id, guacamole_path, guacamole_id = await resources.setup() + token = await get_workspace_owner_token(workspace_id, verify) + sql_path, sql_id = await resources.create(SQL_PAYLOAD, f"/api{workspace_path}/workspace-services", token) + vm_path, vm_id = await resources.create(VM_PAYLOAD, f"/api{guacamole_path}/user-resources", token) + sql = (await get_resource(f"/api{sql_path}", token, verify))["workspaceService"] + vm_properties = (await get_resource(f"/api{vm_path}", token, verify))["userResource"]["properties"] + subscription_id = vm_properties.get("workspace_subscription_id") or os.environ["ARM_SUBSCRIPTION_ID"] + server = f"azsql-{config.TRE_ID}-ws-{workspace_id[-4:]}-svc-{sql_id[-4:]}" + assert sql["properties"]["azuresql_fqdn"] == f"{server}.database.windows.net" + vault = f"kv-{f'{config.TRE_ID}-ws-{workspace_id[-4:]}'[-20:]}" + azure_vm_id = vm_properties["azure_resource_id"] + async with arm_client() as arm: + vm = await arm.request("GET", azure_vm_id, COMPUTE_API) + group = validate_vm(vm, azure_vm_id, subscription_id, config.TRE_ID, workspace_id, guacamole_id, vm_id) + sql_server = await arm.request("GET", f"{group}/providers/Microsoft.Sql/servers/{server}", "2023-08-01") + assert sql_server["properties"]["publicNetworkAccess"] == "Disabled" + assert sql_server["properties"]["minimalTlsVersion"] == "1.2" + vault_id = f"{group}/providers/Microsoft.KeyVault/vaults/{vault}" + secret_name = f"{server}-administrator-password" + probe_id = uuid4().hex + async with secret_access(arm, vault_id, secret_name, vm["identity"]["principalId"], subscription_id): + probe = dict( + server=f"{server}.database.windows.net", + vault_name=vault, + secret_name=secret_name, + probe_id=probe_id, + ) + await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="write", expected_sku="S1") + LOGGER.info("SQL query passed through private DNS at S1") + async with asyncio.timeout(60 * 60): + await post_resource( + {"properties": {"sql_sku": "S2 | 50 DTUs"}}, + f"/api{sql_path}", + token, + verify, + method="PATCH", + etag=sql["_etag"], + ) + await run_probe(arm, azure_vm_id, vm["location"], **probe, phase="read", expected_sku="S2") + LOGGER.info("SQL data survived the S1-to-S2 configuration upgrade") diff --git a/e2e_tests/test_workspace_base.py b/e2e_tests/test_workspace_base.py new file mode 100644 index 0000000000..5c67e6f7a4 --- /dev/null +++ b/e2e_tests/test_workspace_base.py @@ -0,0 +1,82 @@ +"""An opt-in base-workspace case with its own provisioning and cleanup evidence.""" + +import asyncio + +import pytest +from httpx import AsyncClient +from starlette import status + +from e2e_tests.conftest import clean_up_test_workspace, create_or_get_test_workspace +from e2e_tests.helpers import TIMEOUT, get_admin_token, get_auth_header, get_full_endpoint +from e2e_tests.resources import strings +from e2e_tests.resources.workspace import get_workspace, get_workspace_auth_details +from e2e_tests.timeouts import cleanup_deadline + +pytestmark = pytest.mark.asyncio(loop_scope="session") +BODY_TIMEOUT_SECONDS = 5 * 60 +CLEANUP_TIMEOUT_SECONDS = 60 * 60 + + +async def delete_and_check_workspace(workspace_path, verify): + async with cleanup_deadline(CLEANUP_TIMEOUT_SECONDS): + await clean_up_test_workspace(pre_created_workspace_id="", workspace_path=workspace_path, verify=verify) + admin_token = await get_admin_token(verify) + async with AsyncClient(verify=verify) as client: + response = await client.get( + get_full_endpoint(f"/api{workspace_path}"), headers=get_auth_header(admin_token), timeout=TIMEOUT + ) + assert response.status_code == status.HTTP_404_NOT_FOUND, ( + f"Workspace remains visible after uninstall: HTTP {response.status_code}" + ) + + +@pytest.mark.workspace_validation +async def test_base_workspace_lifecycle(verify): + # Always create an owned workspace, even when other suites reuse configured IDs. + workspace_path, workspace_id = await create_or_get_test_workspace( + auth_type="Automatic", verify=verify, template_name=strings.BASE_WORKSPACE, pre_created_workspace_id="" + ) + original = None + try: + async with asyncio.timeout(BODY_TIMEOUT_SECONDS): + admin_token = await get_admin_token(verify) + async with AsyncClient(verify=verify) as client: + workspace = await get_workspace(client, workspace_id, get_auth_header(admin_token)) + assert workspace["id"] == workspace_id + assert workspace["templateName"] == strings.BASE_WORKSPACE + assert workspace["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED + assert workspace["isEnabled"] is True + + workspace_token, _ = await get_workspace_auth_details(admin_token, workspace_id, verify) + response = await client.get( + get_full_endpoint(f"/api{workspace_path}/workspace-services"), + headers=get_auth_header(workspace_token), + timeout=TIMEOUT, + ) + assert response.status_code == status.HTTP_200_OK, ( + f"Workspace identity cannot list services: HTTP {response.status_code}" + ) + assert response.json()["workspaceServices"] == [], "New workspace contains unexpected services" + except BaseException as error: + original = error + raise + finally: + # Caller cancellation must wait for the cleanup task's own finite deadline. + cleanup = asyncio.create_task(delete_and_check_workspace(workspace_path, verify)) + cancelled = None + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError as error: + cancelled = cancelled or error + except Exception: + break + try: + cleanup.result() + except (Exception, asyncio.CancelledError) as cleanup_error: + failure = original or cancelled + if failure is None: + raise + failure.add_note(f"Workspace cleanup also failed: {cleanup_error!r}") + if original is None and cancelled is not None: + raise cancelled diff --git a/e2e_tests/unit_tests/test_base_workspace_lifecycle.py b/e2e_tests/unit_tests/test_base_workspace_lifecycle.py new file mode 100644 index 0000000000..c14c7dc323 --- /dev/null +++ b/e2e_tests/unit_tests/test_base_workspace_lifecycle.py @@ -0,0 +1,234 @@ +"""Exercise the owned base-workspace case through real HTTP helpers without Azure.""" + +import asyncio +import json +import unittest +from unittest.mock import AsyncMock, patch + +from httpx import AsyncClient, MockTransport, Response + +import helpers as short_helpers +from resources import resource as short_resource +from e2e_tests import conftest as fixtures, helpers, test_workspace_base as base +from e2e_tests.resources import resource, workspace + + +WORKSPACE = "11111111-1111-4111-8111-111111111111" +OPERATIONS = { + phase: f"00000000-0000-4000-8000-00000000000{index}" + for index, phase in enumerate(("install", "disable", "delete"), 1) +} + + +class BaseWorkspaceLifecycleTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.yield_to_loop = asyncio.sleep + self.path = f"/workspaces/{WORKSPACE}" + self.requests = [] + self.deleted = False + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + self.enterContext(patch.object(helpers.config, "TEST_WORKSPACE_ID", "existing-unrelated-workspace")) + self.enterContext(patch.object(fixtures, "get_admin_token", AsyncMock(return_value="admin-token"))) + self.enterContext(patch.object(base, "get_admin_token", AsyncMock(return_value="admin-token"))) + self.enterContext(patch.object(fixtures.asyncio, "sleep", AsyncMock())) + self.enterContext(patch.object(workspace, "get_token", return_value="workspace-token")) + self.enterContext( + patch( + "httpx.AsyncHTTPTransport.handle_async_request", + AsyncMock(side_effect=AssertionError("Unexpected live HTTP request in an offline test")), + ) + ) + + def mock_api( + self, + *, + denied=False, + missing_scope=False, + retained=False, + delete_failed=False, + install_failed=False, + rejected=False, + block_services=False, + block_delete=False, + ): + self.service_started = asyncio.Event() + self.delete_started = asyncio.Event() + self.delete_release = asyncio.Event() + + async def handle(request): + self.requests.append(request) + uri = request.url.path + if uri == "/api/workspace-templates/tre-workspace-base": + return Response(200, json={"properties": {"enable_backup": {"type": "boolean"}}}) + if request.method == "POST": + self.assertEqual(uri, "/api/workspaces") + payload = json.loads(request.content) + self.assertEqual(payload["templateName"], "tre-workspace-base") + self.assertEqual(payload["properties"]["auth_type"], "Automatic") + self.assertIs(payload["properties"]["enable_backup"], False) + phase, code, state = "install", 400 if rejected else 202, "deploying" + elif request.method == "PATCH": + self.assertEqual(uri, "/api" + self.path) + self.assertEqual(json.loads(request.content), {"isEnabled": False}) + phase, code, state = "disable", 202, "updating" + elif request.method == "DELETE": + self.assertEqual(uri, "/api" + self.path) + self.delete_started.set() + if block_delete: + await self.delete_release.wait() + phase, code, state = "delete", 200, "deleting" + elif uri.endswith("/workspace-services"): + self.assertEqual(request.headers["authorization"], "Bearer workspace-token") + self.service_started.set() + if block_services: + await asyncio.Event().wait() + return Response(403 if denied else 200, json={"workspaceServices": []}) + elif uri == "/api" + self.path: + self.assertEqual(request.headers["authorization"], "Bearer admin-token") + if self.deleted and not retained: + return Response(404) + return Response( + 200, + json={ + "workspace": { + "id": WORKSPACE, + "templateName": "tre-workspace-base", + "deploymentStatus": "deployed", + "isEnabled": True, + "properties": {} if missing_scope else {"scope_id": "workspace-scope"}, + } + }, + ) + else: + phase = next(name for name, value in OPERATIONS.items() if uri.endswith(value)) + code = 200 + state = { + "install": "deployment_failed" if install_failed else "deployed", + "disable": "updated", + "delete": "deleting_failed" if delete_failed else "deleted", + }[phase] + if state == "deleted": + self.deleted = True + return Response( + code, + headers={"Location": f"/api{self.path}/operations/{OPERATIONS[phase]}"}, + json={ + "operation": { + "resourcePath": self.path, + "resourceId": WORKSPACE, + "status": state, + "message": "Mock result", + "steps": [], + }, + }, + ) + + def client(**_): + return AsyncClient(transport=MockTransport(handle)) + + for module in (base, workspace, resource, short_resource, short_helpers): + self.enterContext(patch.object(module, "AsyncClient", side_effect=client)) + + def assert_owned_workspace_removed(self): + self.assertTrue(self.deleted) + self.assertEqual([str(r.url.path) for r in self.requests if r.method == "DELETE"], ["/api" + self.path]) + self.assertFalse(any("existing-unrelated-workspace" in str(r.url) for r in self.requests)) + + async def test_creates_owned_workspace_uses_workspace_token_and_checks_removal(self): + self.mock_api() + await base.test_base_workspace_lifecycle(True) + self.assert_owned_workspace_removed() + self.assertEqual(self.requests[-1].url.path, "/api" + self.path) + + async def test_failed_access_check_still_removes_the_workspace(self): + self.mock_api(denied=True) + with self.assertRaisesRegex(AssertionError, "cannot list services"): + await base.test_base_workspace_lifecycle(True) + self.assert_owned_workspace_removed() + + async def test_missing_workspace_scope_still_removes_the_workspace(self): + self.mock_api(missing_scope=True) + with self.assertRaisesRegex(Exception, "Scope Id not found"): + await base.test_base_workspace_lifecycle(True) + self.assert_owned_workspace_removed() + + async def test_visible_workspace_after_successful_delete_fails(self): + self.mock_api(retained=True) + with self.assertRaisesRegex(AssertionError, "remains visible after uninstall"): + await base.test_base_workspace_lifecycle(True) + + async def test_cleanup_failure_preserves_the_original_assertion(self): + self.mock_api(denied=True, delete_failed=True) + with self.assertRaisesRegex(AssertionError, "cannot list services") as caught: + await base.test_base_workspace_lifecycle(True) + self.assertIn("Workspace cleanup also failed", "\n".join(caught.exception.__notes__)) + + async def test_failed_accepted_install_is_removed_by_the_existing_recovery_helper(self): + self.mock_api(install_failed=True) + with self.assertRaises(AssertionError): + await base.test_base_workspace_lifecycle(True) + self.assert_owned_workspace_removed() + + async def test_rejected_creation_does_not_delete_any_workspace(self): + self.mock_api(rejected=True) + with self.assertRaises(AssertionError): + await base.test_base_workspace_lifecycle(True) + self.assertFalse(any(r.method == "DELETE" for r in self.requests)) + + async def test_body_timeout_finishes_cleanup(self): + self.mock_api(block_services=True) + with patch.object(base, "BODY_TIMEOUT_SECONDS", 0.05): + with self.assertRaises(TimeoutError): + await base.test_base_workspace_lifecycle(True) + self.assert_owned_workspace_removed() + + async def test_cancellation_during_access_check_finishes_cleanup(self): + self.mock_api(block_services=True) + task = asyncio.create_task(base.test_base_workspace_lifecycle(True)) + await asyncio.wait_for(self.service_started.wait(), 1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await asyncio.wait_for(task, 1) + self.assert_owned_workspace_removed() + + async def test_repeated_cancellation_during_recovery_preserves_original_cancellation(self): + self.mock_api(block_services=True, block_delete=True) + task = asyncio.create_task(base.test_base_workspace_lifecycle(True)) + await asyncio.wait_for(self.service_started.wait(), 1) + task.cancel("body cancelled") + await asyncio.wait_for(self.delete_started.wait(), 1) + for _ in range(2): + task.cancel("cleanup cancelled") + await self.yield_to_loop(0) + self.assertFalse(task.done()) + self.delete_release.set() + with self.assertRaises(asyncio.CancelledError) as caught: + await asyncio.wait_for(task, 1) + self.assertEqual(caught.exception.args, ("body cancelled",)) + self.assert_owned_workspace_removed() + + async def test_cancellation_during_cleanup_after_body_success_finishes_deletion(self): + self.mock_api(block_delete=True) + task = asyncio.create_task(base.test_base_workspace_lifecycle(True)) + await asyncio.wait_for(self.delete_started.wait(), 1) + task.cancel("cleanup cancelled") + await self.yield_to_loop(0) + task.cancel("cleanup cancelled again") + await self.yield_to_loop(0) + self.assertFalse(task.done()) + self.delete_release.set() + with self.assertRaises(asyncio.CancelledError) as caught: + await asyncio.wait_for(task, 1) + self.assertEqual(caught.exception.args, ("cleanup cancelled",)) + self.assert_owned_workspace_removed() + + async def test_cancelled_cleanup_keeps_its_deadline_and_reports_timeout(self): + self.mock_api(block_delete=True) + with patch.object(base, "CLEANUP_TIMEOUT_SECONDS", 0.05): + task = asyncio.create_task(base.test_base_workspace_lifecycle(True)) + await asyncio.wait_for(self.delete_started.wait(), 1) + task.cancel("cleanup cancelled") + with self.assertRaises(asyncio.CancelledError) as caught: + await asyncio.wait_for(task, 1) + self.assertIn("TimeoutError", "\n".join(caught.exception.__notes__)) + self.assertFalse(self.deleted) diff --git a/e2e_tests/unit_tests/test_bundle_selection.py b/e2e_tests/unit_tests/test_bundle_selection.py index e69e9d1f8f..fdca790ea5 100644 --- a/e2e_tests/unit_tests/test_bundle_selection.py +++ b/e2e_tests/unit_tests/test_bundle_selection.py @@ -15,7 +15,7 @@ ROOT = run_bundle.REPO_ROOT SQL = "tre-workspace-service-azuresql" -SQL_CASE = f"test_workspace_services.py::test_install_workspace_service[{SQL}]" +SQL_CASE = "test_azuresql.py::test_sql_query_survives_sku_upgrade" class BundleSelectionTests(unittest.TestCase): @@ -94,7 +94,7 @@ def test_every_declared_case_collects_from_current_source(self): collected = sorted(line for line in result.stdout.splitlines() if line.startswith("test_") and "::" in line) self.assertEqual(collected, expected) - def test_sql_collects_only_its_named_parametrised_case(self): + def test_sql_collects_only_its_named_lifecycle_case(self): result, report = self.run_selector( "--bundle", SQL, "--collect-only", environment={"TEST_WORKSPACE_ID": "existing-workspace-id"} ) @@ -136,6 +136,20 @@ def test_unrelated_selection_does_not_reject_a_configured_aad_workspace(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(report["collected_tests"], [SQL_CASE]) + def test_base_workspace_collects_its_own_case_without_reusing_a_workspace(self): + result, report = self.run_selector( + "--bundle", + "tre-workspace-base", + "--collect-only", + environment={"TEST_WORKSPACE_ID": "existing-unrelated-workspace"}, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(report["collected_tests"], ["test_workspace_base.py::test_base_workspace_lifecycle"]) + self.assertEqual(report["status"], "not_run") + self.assertFalse(report["full_lifecycle_proven"]) + self.assertEqual(report["resources"], []) + self.assertEqual(report["declared_prerequisites"], {}) + def test_unknown_partial_and_shell_like_identifiers_fail_before_collection(self): for identifier in ("azuresql", "tre-nonexistent", SQL + "; echo unexpected", SQL + "$(echo unexpected)"): with self.subTest(identifier=identifier): diff --git a/e2e_tests/unit_tests/test_polling_diagnostics.py b/e2e_tests/unit_tests/test_polling_diagnostics.py new file mode 100644 index 0000000000..8451030d08 --- /dev/null +++ b/e2e_tests/unit_tests/test_polling_diagnostics.py @@ -0,0 +1,132 @@ +"""Keep useful HTTP failure evidence without exposing response data or credentials.""" + +import unittest +from unittest.mock import patch + +from httpx import AsyncClient, MockTransport, Response + +from e2e_tests import helpers +from e2e_tests.resources import deployment, strings + + +WORKSPACE = "9f293650-29c7-4291-a074-c5c0aaedd7b7" +OPERATION = "2913b8be-2594-4088-8d93-42a296fe69cc" +REQUEST_ID = "05c22c61-9d22-49d7-b333-65b86a0d70dd" +TRACE = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" +ENDPOINT = f"/api/workspaces/{WORKSPACE}/operations/{OPERATION}" + + +class PollingDiagnosticsTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.object(helpers.config, "TRE_URL", "https://tre.example.test")) + + async def failure_log(self, response, endpoint=ENDPOINT): + requests = [] + + def handle(request): + requests.append(request) + return response + + with self.assertLogs(deployment.LOGGER, level="ERROR") as logs: + async with AsyncClient(transport=MockTransport(handle)) as client: + with self.assertRaisesRegex(Exception, "Non 200 response in check_deployment"): + await deployment.check_deployment(client, endpoint, "request-token-secret") + self.assertEqual(len(requests), 1, "An HTTP failure must not trigger a retry") + return "\n".join(logs.output) + + async def test_records_utc_time_path_status_and_valid_correlation_ids(self): + log = await self.failure_log( + Response( + 500, + content=b"Internal Server Error", + headers={ + "content-type": "text/plain; charset=utf-8", + "x-ms-request-id": REQUEST_ID, + "traceparent": TRACE, + }, + ) + ) + for value in (ENDPOINT, "500", "GET", "text/plain", REQUEST_ID, TRACE): + self.assertIn(value, log) + self.assertRegex(log, r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+00:00") + self.assertIn("body_bytes", log) + self.assertIn("body_kind", log) + + async def test_excludes_body_query_fragment_credentials_and_arbitrary_headers(self): + secret = "sensitive-payload-secret" + log = await self.failure_log( + Response( + 500, + json={"error": secret}, + headers={ + "set-cookie": secret, + "authorization": secret, + "x-unknown-header": secret, + "x-ms-request-id": secret, + }, + ), + ENDPOINT + "?sig=query-secret#fragment-secret", + ) + self.assertIn(ENDPOINT, log) + for value in (secret, "query-secret", "fragment-secret", "request-token-secret", "tre.example.test"): + self.assertNotIn(value, log) + self.assertIn("json_like", log) + + async def test_suppresses_unknown_paths_and_unbounded_header_values(self): + log = await self.failure_log( + Response( + 500, + content=b"body-secret" * 1000, + headers={ + "content-type": "secret-media-type" * 1000, + "x-ms-correlation-request-id": "a" * 10000, + "request-id": "line-one\nline-two", + "traceparent": "bad-trace-secret", + }, + ), + "/api/operations/path-secret", + ) + self.assertIn("", log) + for value in ("path-secret", "secret-media-type", "line-one", "line-two", "bad-trace-secret", "body-secret"): + self.assertNotIn(value, log) + self.assertLess(len(log), 1200) + + async def test_classifies_response_without_copying_its_content(self): + for content, content_type, expected in ( + (b"", "", "empty"), + (b" failure", "text/html", "html_like"), + (b"[not even valid json", "application/json", "json_like"), + (b"Internal Server Error", "text/plain", "other"), + (b"\x00\xff", "application/octet-stream", "other"), + ): + with self.subTest(expected=expected, content_type=content_type): + log = await self.failure_log(Response(500, content=content, headers={"content-type": content_type})) + self.assertIn(expected, log) + + async def test_other_http_errors_still_fail_once(self): + for code in (400, 401, 403, 429, 503): + with self.subTest(code=code): + log = await self.failure_log(Response(code)) + self.assertIn(str(code), log) + + async def test_workspace_service_and_shared_service_operation_paths_are_retained(self): + for endpoint in ( + f"/api/operations/{OPERATION}", + f"/api/shared-services/{WORKSPACE}/operations/{OPERATION}", + f"/api/workspaces/{WORKSPACE}/workspace-services/{WORKSPACE}/operations/{OPERATION}", + f"/api/workspaces/{WORKSPACE}/workspace-services/{WORKSPACE}/user-resources/{WORKSPACE}/operations/{OPERATION}", + ): + with self.subTest(endpoint=endpoint): + log = await self.failure_log(Response(500), endpoint) + self.assertIn(endpoint, log) + + async def test_success_and_deletion_keep_the_existing_results(self): + for response, expected in ( + (Response(200, json={"operation": {"status": "deleting", "message": "waiting", "steps": []}}), "deleting"), + (Response(404), strings.RESOURCE_STATUS_DELETED), + ): + with self.subTest(expected=expected), patch.object(deployment.LOGGER, "error") as log: + async with AsyncClient(transport=MockTransport(lambda _: response)) as client: + state, _, _ = await deployment.check_deployment(client, ENDPOINT, "token") + self.assertEqual(state, expected) + log.assert_not_called() diff --git a/e2e_tests/unit_tests/test_sql_deadlines.py b/e2e_tests/unit_tests/test_sql_deadlines.py new file mode 100644 index 0000000000..4bcdd4b23e --- /dev/null +++ b/e2e_tests/unit_tests/test_sql_deadlines.py @@ -0,0 +1,228 @@ +"""Exercise SQL setup, cancellation and nested cleanup with scaled deadlines.""" + +import asyncio +from builtins import ExceptionGroup +import os +import time +import traceback +import unittest +from contextlib import ExitStack +from unittest.mock import AsyncMock, patch + +from e2e_tests.resources import sql_lifecycle as lifecycle +from e2e_tests.resources import resource +from httpx import AsyncClient, MockTransport, Response +from e2e_tests import helpers + + +class SQLDeadlineTests(unittest.IsolatedAsyncioTestCase): + def budgets(self, *, total=0.5, reserve=0.4): + patches = ExitStack() + patches.enter_context(patch.object(lifecycle, "LIFECYCLE_SECONDS", total)) + patches.enter_context(patch.object(lifecycle, "CLEANUP_RESERVE_SECONDS", reserve)) + patches.enter_context(patch.dict(os.environ, SQL_VALIDATION_DEADLINE="")) + return patches + + async def test_slow_body_reserves_cleanup_for_all_four_owned_resources(self): + events = [] + + async def delete(name): + events.append(name + " start") + await asyncio.sleep(0.03) + events.append(name + " done") + + with self.budgets(): + with self.assertRaises(TimeoutError): + async with lifecycle.sql_lifecycle(True) as owned: + for name in ("workspace", "guacamole", "sql", "vm"): + owned.own(name, delete, name) + await asyncio.sleep(10) + self.assertEqual( + events, + [f"{name} {state}" for name in ("vm", "sql", "guacamole", "workspace") for state in ("start", "done")], + ) + + async def test_slow_delete_leaves_time_for_each_parent_and_reports_original_failure(self): + events = [] + + async def delete(name): + events.append(name) + if name == "vm": + await asyncio.sleep(10) + + with self.budgets(total=0.2, reserve=0.15): + with self.assertRaisesRegex(RuntimeError, "body failed") as caught: + async with lifecycle.sql_lifecycle(True) as owned: + for name in ("workspace", "guacamole", "sql", "vm"): + owned.own(name, delete, name) + raise RuntimeError("body failed") + self.assertEqual(events, ["vm", "sql", "guacamole", "workspace"]) + self.assertIn("TimeoutError", " ".join(caught.exception.__notes__)) + self.assertIn("vm", " ".join(caught.exception.__notes__)) + + async def test_repeated_cancellation_waits_for_cleanup(self): + started = asyncio.Event() + finished = asyncio.Event() + + async def delete(): + started.set() + await asyncio.sleep(0.06) + finished.set() + + async def run(): + async with lifecycle.sql_lifecycle(True) as owned: + owned.own("vm", delete) + + with self.budgets(): + task = asyncio.create_task(run()) + await started.wait() + task.cancel() + await asyncio.sleep(0.01) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertTrue(finished.is_set()) + + async def test_setup_uses_work_deadline_and_removes_created_workspace(self): + deleted = AsyncMock() + setup_started = asyncio.Event() + + async def create_service(*args): + setup_started.set() + await asyncio.sleep(10) + + with ( + self.budgets(), + patch.object(lifecycle.config, "TEST_WORKSPACE_ID", ""), + patch.object(lifecycle.config, "TEST_WORKSPACE_SERVICE_ID", ""), + patch.object(lifecycle, "create_or_get_test_workspace", AsyncMock(return_value=("/workspaces/ws", "ws"))), + patch.object(lifecycle, "get_workspace_owner_token", AsyncMock(return_value="token")), + patch.object(lifecycle, "create_or_get_test_workpace_service", create_service), + patch.object(lifecycle, "clean_up_test_workspace", deleted), + ): + with self.assertRaises(TimeoutError): + async with lifecycle.sql_lifecycle(True) as owned: + await owned.setup() + self.assertTrue(setup_started.is_set()) + deleted.assert_awaited_once_with("", "/workspaces/ws", True) + + async def test_existing_workspace_and_service_are_not_owned(self): + with ( + self.budgets(), + patch.object(lifecycle.config, "TEST_WORKSPACE_ID", "existing"), + patch.object(lifecycle.config, "TEST_WORKSPACE_SERVICE_ID", "existing-service"), + patch.object(lifecycle, "create_or_get_test_workspace", AsyncMock(return_value=("/workspaces/ws", "ws"))), + patch.object(lifecycle, "get_workspace_owner_token", AsyncMock(return_value="token")), + patch.object( + lifecycle, "create_or_get_test_workpace_service", AsyncMock(return_value=("/service", "service")) + ), + ): + async with lifecycle.sql_lifecycle(True) as owned: + await owned.setup() + self.assertEqual(owned.finalisers, []) + + async def test_job_start_deadline_reduces_work_time(self): + with patch.dict(os.environ, SQL_VALIDATION_DEADLINE=str(time.time() + 125 * 60)): + work, finish = lifecycle.lifecycle_deadlines() + self.assertAlmostEqual(work - asyncio.get_running_loop().time(), 5 * 60, delta=1) + self.assertAlmostEqual(finish - work, 120 * 60, delta=1) + + async def test_job_start_deadline_refuses_late_start_before_creating_resources(self): + for value in (str(time.time() + 60), str(time.time() - 1), "nan", "inf", "invalid"): + with self.subTest(value=value), patch.dict(os.environ, SQL_VALIDATION_DEADLINE=value): + with self.assertRaises((ValueError, TimeoutError)): + async with lifecycle.sql_lifecycle(True): + self.fail("A late or invalid job deadline must not start resource creation") + + async def test_failed_accepted_create_finishes_recovery_before_parent_cleanup(self): + events = [] + states = iter(("deploying", "deployed")) + child = "/workspaces/ws/workspace-services/sql" + + async def handle(request): + phase = request.url.path.rsplit("/", 1)[-1] + state, code = "deployed", 200 + if request.method == "POST": + phase, state, code = "install", "deploying", 202 + elif request.method == "PATCH": + phase, state, code = "disable", "updating", 202 + elif request.method == "DELETE": + events.append("child delete start") + await asyncio.sleep(0.03) + events.append("child delete done") + phase, state = "delete", "deleting" + elif phase == "install": + state = next(states) + else: + state = {"disable": "updated", "delete": "deleted"}[phase] + return Response( + code, + headers={"Location": f"/api/operations/{phase}"}, + json={ + "operation": { + "resourcePath": child, + "resourceId": "sql", + "status": state, + "message": "operation", + "steps": [], + } + }, + ) + + async def parent(): + events.append("parent delete") + + with ( + self.budgets(), + patch.object(helpers.config, "TRE_URL", "https://tre.example.test"), + patch.object(resource, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(handle))), + ): + with self.assertRaises(TimeoutError): + async with lifecycle.sql_lifecycle(True) as owned: + owned.own("workspace", parent) + await owned.create({}, "/api/workspaces/ws/workspace-services", "token") + self.assertEqual(events, ["child delete start", "child delete done", "parent delete"]) + + async def test_cleanup_failure_after_success_fails_the_case(self): + with self.budgets(): + with self.assertRaises(ExceptionGroup) as caught: + async with lifecycle.sql_lifecycle(True) as owned: + owned.own("vm", AsyncMock(side_effect=RuntimeError("DELETE_FAILED"))) + self.assertIn("DELETE_FAILED", str(caught.exception.exceptions[0])) + + async def test_absolute_deadline_bounds_failed_create_recovery(self): + path = "/workspaces/ws/workspace-services/sql" + parent = AsyncMock() + + async def handle(request): + return Response( + 202 if request.method == "POST" else 200, + headers={"Location": "/api/operations/install"}, + json={ + "operation": { + "resourcePath": path, + "resourceId": "sql", + "status": "deploying", + "message": "operation", + "steps": [], + } + }, + ) + + started = asyncio.get_running_loop().time() + with ( + self.budgets(total=0.15, reserve=0.1), + patch.object(helpers.config, "TRE_URL", "https://tre.example.test"), + patch.object(resource, "AsyncClient", side_effect=lambda **_: AsyncClient(transport=MockTransport(handle))), + ): + with self.assertRaises(TimeoutError) as caught: + async with lifecycle.sql_lifecycle(True) as owned: + owned.own("workspace", parent) + await owned.create({}, "/api/workspaces/ws/workspace-services", "token") + self.assertLess(asyncio.get_running_loop().time() - started, 1) + # Expiry is explicit. No late cleanup task can outlive the SQL case. + parent.assert_not_awaited() + details = "".join(traceback.format_exception(caught.exception)) + self.assertIn("SQL validation cleanup failed for workspace", details) + self.assertIn(path, details) + self.assertEqual([task for task in asyncio.all_tasks() if task is not asyncio.current_task()], []) diff --git a/e2e_tests/unit_tests/test_sql_lifecycle.py b/e2e_tests/unit_tests/test_sql_lifecycle.py new file mode 100644 index 0000000000..88b58e9024 --- /dev/null +++ b/e2e_tests/unit_tests/test_sql_lifecycle.py @@ -0,0 +1,126 @@ +"""Exercise SQL lifecycle orchestration against the exposed VM API properties.""" + +from contextlib import asynccontextmanager +import os +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, patch + +from e2e_tests import test_azuresql as sql +from e2e_tests.resources.sql_probe import SECRETS_USER +from e2e_tests.resources.sql_lifecycle import SQLResources + + +class SQLLifecycleTests(unittest.IsolatedAsyncioTestCase): + async def run_lifecycle(self, workspace_subscription=None, *, unrelated_vm=False): + workspace = ("/workspaces/ws1234", "ws1234", "/workspaces/ws1234/workspace-services/guac", "guac") + subscription = workspace_subscription or "sub" + group = f"/subscriptions/{subscription}/resourceGroups/rg-tre-test-ws-1234" + vm_id = f"{group}/providers/Microsoft.Compute/virtualMachines/vm" + properties = {"azure_resource_id": vm_id} + if workspace_subscription is not None: + properties["workspace_subscription_id"] = workspace_subscription + server = "azsql-tre-test-ws-1234-svc-5678" + vm = { + "id": vm_id, + "location": "switzerlandnorth", + "identity": {"principalId": "principal"}, + "tags": { + "tre_id": "tre-test", + "tre_workspace_id": "another-workspace" if unrelated_vm else "ws1234", + "tre_workspace_service_id": "guac", + "tre_user_resource_id": "vm", + }, + } + arm = SimpleNamespace( + request=AsyncMock( + side_effect=[ + vm, + {"properties": {"publicNetworkAccess": "Disabled", "minimalTlsVersion": "1.2"}}, + {}, + ] + ), + delete=AsyncMock(), + ) + resources = [] + + async def record_cleanup(name): + resources.append(name) + + async def resource(owner, payload, endpoint, token): + name = "sql5678" if payload is sql.SQL_PAYLOAD else "vm" + owner.own(name, record_cleanup, name) + return f"{endpoint.removeprefix('/api')}/{name}", name + + @asynccontextmanager + async def arm_client(): + yield arm + + with ( + patch.dict(os.environ, ARM_SUBSCRIPTION_ID="sub", AZURE_ENVIRONMENT="AzureCloud"), + patch.object(sql.config, "TRE_ID", "tre-test"), + patch.object(SQLResources, "create", resource), + patch.object(SQLResources, "setup", AsyncMock(return_value=workspace)), + patch.object(sql, "get_workspace_owner_token", AsyncMock(return_value="token")), + patch.object( + sql, + "get_resource", + AsyncMock( + side_effect=[ + { + "workspaceService": { + "properties": {"azuresql_fqdn": f"{server}.database.windows.net"}, + "_etag": "etag", + } + }, + {"userResource": {"properties": properties}}, + ] + ), + ), + patch.object(sql, "arm_client", arm_client), + patch.object(sql, "post_resource", AsyncMock()) as update, + patch.object(sql, "run_probe", AsyncMock()) as probe, + ): + if unrelated_vm: + with self.assertRaisesRegex(ValueError, "does not match the test-created resource"): + await sql.test_sql_query_survives_sku_upgrade(True) + probe.assert_not_awaited() + self.assertEqual(arm.request.await_count, 1) + arm.delete.assert_not_awaited() + self.assertEqual(resources, ["vm", "sql5678"]) + return + await sql.test_sql_query_survives_sku_upgrade(True) + self.assertEqual([call.kwargs["phase"] for call in probe.await_args_list], ["write", "read"]) + self.assertEqual([call.kwargs["expected_sku"] for call in probe.await_args_list], ["S1", "S2"]) + self.assertEqual({call.kwargs["vault_name"] for call in probe.await_args_list}, {"kv-tre-test-ws-1234"}) + self.assertEqual(arm.request.await_args_list[1].args[1], f"{group}/providers/Microsoft.Sql/servers/{server}") + grant = arm.request.await_args_list[2] + self.assertEqual(grant.args[0], "PUT") + self.assertTrue( + grant.args[1].startswith( + f"{group}/providers/Microsoft.KeyVault/vaults/kv-tre-test-ws-1234/secrets/" + f"{server}-administrator-password/providers/Microsoft.Authorization/roleAssignments/" + ) + ) + self.assertEqual( + grant.kwargs["body"]["properties"], + { + "roleDefinitionId": f"/subscriptions/{subscription}/providers/Microsoft.Authorization/roleDefinitions/{SECRETS_USER}", + "principalId": "principal", + "principalType": "ServicePrincipal", + }, + ) + self.assertEqual(arm.delete.await_args.args[0], grant.args[1]) + self.assertEqual(update.call_args.args[0], {"properties": {"sql_sku": "S2 | 50 DTUs"}}) + self.assertEqual(resources, ["vm", "sql5678"]) + + async def test_probes_use_workspace_vault_without_a_vm_keyvault_property(self): + await self.run_lifecycle() + + async def test_workspace_subscription_selects_vm_and_secret_role(self): + for subscription in ("", "sub", "workspace-sub"): + with self.subTest(subscription=subscription): + await self.run_lifecycle(subscription) + + async def test_unrelated_vm_is_rejected_before_granting_secret_access(self): + await self.run_lifecycle("workspace-sub", unrelated_vm=True) diff --git a/e2e_tests/unit_tests/test_sql_probe.py b/e2e_tests/unit_tests/test_sql_probe.py new file mode 100644 index 0000000000..f3e4ac5cd7 --- /dev/null +++ b/e2e_tests/unit_tests/test_sql_probe.py @@ -0,0 +1,293 @@ +"""Check the SQL probe's evidence, ownership and recovery boundaries.""" + +from copy import deepcopy +import asyncio +import json +import os +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from httpx import AsyncClient, MockTransport, Response +from jsonschema import validate + +from api_app.services.schema_service import enrich_user_resource_template, enrich_workspace_service_template +from e2e_tests import test_azuresql as sql +from e2e_tests.resources import sql_probe as probe +from e2e_tests.resources import sql_lifecycle as lifecycle + + +ROOT = Path(__file__).resolve().parents[2] +GROUP = "/subscriptions/sub/resourceGroups/rg-tretest-ws-abcd" +VM_ID = f"{GROUP}/providers/Microsoft.Compute/virtualMachines/windowsvm1234" + + +def result(**overrides): + evidence = {"phase": "read", "probe": "abc", "sku": "S2", "row_count": 1, "private_dns": True, **overrides} + return { + "properties": { + "provisioningState": "Succeeded", + "instanceView": { + "executionState": "Succeeded", + "exitCode": 0, + "output": "SQL_PROBE_RESULT=" + json.dumps(evidence), + }, + } + } + + +class ProbeEvidenceTests(unittest.TestCase): + def test_success_requires_expected_persisted_data_and_sku(self): + self.assertTrue(probe.probe_result(result(), "read", "abc", "S2")) + for change in ({"phase": "write"}, {"probe": "other"}, {"sku": "S1"}, {"row_count": 0}, {"private_dns": False}): + with self.subTest(change=change), self.assertRaises(AssertionError): + probe.probe_result(result(**change), "read", "abc", "S2") + + def test_provisioning_success_does_not_prove_command_success(self): + resource = result() + resource["properties"]["instanceView"] = {} + self.assertFalse(probe.probe_result(resource, "read", "abc", "S2")) + for state, code in (("Failed", 1), ("TimedOut", -1), ("Succeeded", 1)): + resource["properties"]["instanceView"] = {"executionState": state, "exitCode": code} + with self.subTest(state=state), self.assertRaises(AssertionError): + probe.probe_result(resource, "read", "abc", "S2") + + def test_vm_must_belong_to_this_test_workspace_and_user_resource(self): + vm = { + "id": VM_ID, + "tags": { + "tre_id": "tretest", + "tre_workspace_id": "ws-abcd", + "tre_workspace_service_id": "guac", + "tre_user_resource_id": "vm", + }, + "identity": {"principalId": "principal"}, + } + args = (VM_ID, "sub", "tretest", "ws-abcd", "guac", "vm") + self.assertEqual(probe.validate_vm(vm, *args), GROUP) + for tag in vm["tags"]: + changed = deepcopy(vm) + changed["tags"][tag] = "unrelated" + with self.subTest(tag=tag), self.assertRaises(ValueError): + probe.validate_vm(changed, *args) + with self.assertRaises(ValueError): + probe.validate_vm(vm, VM_ID, "another-subscription", *args[2:]) + + def test_payloads_match_enriched_api_schemas(self): + cases = ( + (sql.SQL_PAYLOAD, "azuresql", enrich_workspace_service_template), + (sql.VM_PAYLOAD, "guacamole/user_resources/guacamole-azure-windowsvm", enrich_user_resource_template), + ) + for payload, folder, enrich in cases: + with self.subTest(bundle=payload["templateName"]): + schema = json.loads( + (ROOT / "templates/workspace_services" / folder / "template_schema.json").read_text() + ) + validate(payload["properties"], enrich(SimpleNamespace(model_dump=lambda **_: schema))) + self.assertFalse(any(v for k, v in sql.VM_PAYLOAD["properties"].items() if k.startswith("install_"))) + + def test_oidc_bearer_is_never_sent_to_an_untrusted_endpoint(self): + with patch.dict(os.environ, {"ACTIONS_ID_TOKEN_REQUEST_URL": "https://example.test/token"}): + with patch.object(probe.requests, "get") as get, self.assertRaises(ValueError): + probe.github_assertion() + get.assert_not_called() + + def test_oidc_request_refreshes_assertion_and_disallows_redirects(self): + environment = { + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://test.actions.githubusercontent.com/token?job=1", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + } + response = Mock(status_code=200) + response.json.return_value = {"value": "assertion"} + with patch.dict(os.environ, environment), patch.object(probe.requests, "get", return_value=response) as get: + self.assertEqual(probe.github_assertion(), "assertion") + self.assertEqual(probe.github_assertion(), "assertion") + self.assertEqual(get.call_count, 2) + self.assertFalse(get.call_args.kwargs["allow_redirects"]) + self.assertEqual(get.call_args.kwargs["params"], {"audience": "api://AzureADTokenExchange"}) + + +class ProbeRecoveryTests(unittest.IsolatedAsyncioTestCase): + async def test_only_the_temporary_secret_scoped_assignment_is_deleted(self): + arm = SimpleNamespace(request=AsyncMock(return_value={}), delete=AsyncMock()) + vault = f"{GROUP}/providers/Microsoft.KeyVault/vaults/kv-test" + + async def failing_query(): + async with probe.secret_access(arm, vault, "sql-password", "principal", "sub"): + raise RuntimeError("query failed") + + with self.assertRaisesRegex(RuntimeError, "query failed"): + await failing_query() + call = arm.request.call_args + assignment = call.args[1] + self.assertTrue( + assignment.startswith(vault + "/secrets/sql-password/providers/Microsoft.Authorization/roleAssignments/") + ) + self.assertEqual(call.kwargs["body"]["properties"]["principalId"], "principal") + self.assertTrue(call.kwargs["body"]["properties"]["roleDefinitionId"].endswith(probe.SECRETS_USER)) + arm.delete.assert_awaited_once_with(assignment, probe.ROLE_API) + + async def test_assignment_is_cleaned_if_create_response_is_lost(self): + arm = SimpleNamespace(request=AsyncMock(side_effect=TimeoutError("response lost")), delete=AsyncMock()) + with self.assertRaises(TimeoutError): + async with probe.secret_access(arm, GROUP + "/vault", "secret", "principal", "sub"): + self.fail("Creation should fail") + arm.delete.assert_awaited_once() + + async def test_failed_command_is_removed_and_never_counts_as_pass(self): + failed = {"properties": {"instanceView": {"executionState": "Failed", "exitCode": 1}}} + arm = SimpleNamespace(request=AsyncMock(side_effect=[{}, failed]), delete=AsyncMock()) + with self.assertRaises(AssertionError): + await probe.run_probe( + arm, + VM_ID, + "switzerlandnorth", + server="sql.database.windows.net", + vault_name="kv-test", + secret_name="sql-password", + probe_id="abc", + phase="read", + expected_sku="S2", + ) + command = arm.request.call_args_list[0].args[1] + arm.delete.assert_awaited_once_with(command, probe.COMPUTE_API) + payload = arm.request.call_args_list[0].kwargs["body"] + self.assertFalse(any("password" in p["name"].lower() for p in payload["properties"]["parameters"])) + + async def test_arm_deletion_waits_until_resource_is_gone(self): + calls = [] + + def handle(request): + calls.append(request.method) + return Response(202) if request.method == "DELETE" else Response(404) + + credential = Mock() + credential.get_token.return_value = SimpleNamespace(token="offline") + async with AsyncClient(transport=MockTransport(handle)) as client: + await probe.ArmClient(client, credential).delete(VM_ID + "/runCommands/probe", probe.COMPUTE_API) + self.assertEqual(calls, ["DELETE", "GET"]) + + async def test_cleanup_waits_for_active_update_before_disable(self): + events = [] + + async def get(*_): + events.append("poll") + return {"operations": [{"status": "updating" if len(events) == 1 else "updating_failed"}]} + + async def delete(*_, **__): + events.append("delete") + + async def timed_out_update(): + async with lifecycle.sql_lifecycle(True) as owned: + await owned.create({}, "/services", "token") + raise TimeoutError("update") + + with ( + patch.object(lifecycle, "post_resource", AsyncMock(return_value=("/sql", "id"))), + patch.object(lifecycle, "get_resource", side_effect=get), + patch.object(lifecycle, "disable_and_delete_resource", side_effect=delete), + patch.object(sql.asyncio, "sleep", AsyncMock()), + ): + with self.assertRaisesRegex(TimeoutError, "update"): + await timed_out_update() + self.assertEqual(events, ["poll", "poll", "delete"]) + + async def test_cleanup_failure_preserves_original_query_error(self): + async def failing_query(): + async with lifecycle.sql_lifecycle(True) as owned: + await owned.create({}, "/services", "token") + raise ValueError("query") + + with ( + patch.object(lifecycle, "post_resource", AsyncMock(return_value=("/sql", "id"))), + patch.object(lifecycle, "get_resource", AsyncMock(return_value={"operations": []})), + patch.object(lifecycle, "disable_and_delete_resource", AsyncMock(side_effect=RuntimeError("cleanup"))), + ): + with self.assertRaisesRegex(ValueError, "query") as raised: + await failing_query() + self.assertIn("cleanup", raised.exception.__notes__[0]) + + async def test_work_deadline_during_role_revocation_preserves_query_failure_and_waits(self): + events = [] + + async def delete(*args): + events.append("revoke start") + await asyncio.sleep(0.1) + events.append("revoke done") + + async def delete_vm(): + events.append("vm delete") + + arm = SimpleNamespace(request=AsyncMock(), delete=delete) + with ( + patch.object(lifecycle, "LIFECYCLE_SECONDS", 0.5), + patch.object(lifecycle, "CLEANUP_RESERVE_SECONDS", 0.45), + patch.dict(os.environ, SQL_VALIDATION_DEADLINE=""), + ): + with self.assertRaisesRegex(RuntimeError, "query failed"): + async with lifecycle.sql_lifecycle(True) as owned: + owned.own("vm", delete_vm) + async with probe.secret_access(arm, GROUP + "/vault", "secret", "principal", "sub"): + raise RuntimeError("query failed") + self.assertEqual(events, ["revoke start", "revoke done", "vm delete"]) + + async def test_repeated_cancellation_waits_for_command_removal(self): + started = asyncio.Event() + removed = asyncio.Event() + + async def delete(*args): + started.set() + await asyncio.sleep(0.05) + removed.set() + + arm = SimpleNamespace(request=AsyncMock(side_effect=RuntimeError("command failed")), delete=delete) + task = asyncio.create_task( + probe.run_probe( + arm, + VM_ID, + "switzerlandnorth", + server="sql.database.windows.net", + vault_name="kv-test", + secret_name="secret", + probe_id="abc", + phase="read", + expected_sku="S2", + ) + ) + await started.wait() + task.cancel() + await asyncio.sleep(0.01) + task.cancel() + with self.assertRaisesRegex(RuntimeError, "command failed"): + await task + self.assertTrue(removed.is_set()) + + async def test_work_deadline_during_successful_command_cleanup_still_waits(self): + events = [] + + async def delete(*args): + events.append("command delete start") + await asyncio.sleep(0.1) + events.append("command delete done") + + arm = SimpleNamespace(request=AsyncMock(side_effect=[{}, result()]), delete=delete) + with ( + patch.object(lifecycle, "LIFECYCLE_SECONDS", 0.5), + patch.object(lifecycle, "CLEANUP_RESERVE_SECONDS", 0.45), + patch.dict(os.environ, SQL_VALIDATION_DEADLINE=""), + ): + with self.assertRaises(TimeoutError): + async with lifecycle.sql_lifecycle(True): + await probe.run_probe( + arm, + VM_ID, + "switzerlandnorth", + server="sql.database.windows.net", + vault_name="kv-test", + secret_name="secret", + probe_id="abc", + phase="read", + expected_sku="S2", + ) + self.assertEqual(events, ["command delete start", "command delete done"]) From e4a2879c5eab602f6ccf2513e064b27a6522cc2e Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 11:42:43 +0100 Subject: [PATCH 08/17] Verify Airlock draft deletion with independent storage checks (#5119) (cherry picked from commit ac55e6579f5289212ba7892fe815fca2a7aa5f56) --- CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 12 ++ e2e_tests/airlock/request.py | 71 +++++++++-- e2e_tests/test_airlock.py | 11 +- e2e_tests/unit_tests/test_airlock_request.py | 117 ++++++++++++++++++- 5 files changed, 195 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5ffb4133a..e58d781a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Verify Airlock draft-container deletion through independent storage checks and rejected SAS writes. ([#5258](https://github.com/microsoft/AzureTRE/pull/5258)) * Use supported review VM sizes and validate Windows tools. Forward explicit Nexus consent and bound Airlock and shared-service resource cleanup. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Wait for private certificate storage readiness, reuse trusted certificates and verify TLS for E2E identity authentication. ([#5254](https://github.com/microsoft/AzureTRE/pull/5254)) * Correct managed-identity login in bundle runtimes. Preserve failed uninstall results, allow cleanup when Porter confirms that an installation is absent, and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 528c251801..381a8279f9 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -36,6 +36,18 @@ If a poll fails, use its timestamp, operation ID and available correlation IDs t Check API exception telemetry before a validation run. Record terminal resource deletion separately from the failed poll. These diagnostics help investigate the failure. They do not establish its cause or guarantee that server telemetry is available. +## Airlock draft deletion + +The draft-deletion case requires AzureCloud, `ARM_SUBSCRIPTION_ID`, `TRE_ID` and the existing CI Azure login. +The CI identity must have ARM read access to blob containers in the TRE Airlock storage account. +Local runs use the Azure CLI login. CI uses its existing federated identity. The test does not add permissions. + +After submission, the test checks that the old draft SAS cannot write a new blob. +A `ContainerNotFound` or `AuthorizationPermissionMismatch` response must also have independent ARM evidence: +the draft is absent, and the submitted container exists with matching workspace and import-stage metadata. +A permission error alone cannot pass the test. Other storage errors and ARM lookup failures remain test failures. +These checks share a 60-second deadline. Submitted file contents are checked separately by the review-VM lifecycle test. + ## Validate one bundle The [bundle coverage map](https://github.com/microsoft/AzureTRE/blob/main/e2e_tests/bundle_coverage.json) lists every Porter bundle, its prerequisite bundles, exact pytest cases and remaining coverage gaps. diff --git a/e2e_tests/airlock/request.py b/e2e_tests/airlock/request.py index 46fadad84e..df162c3f0c 100644 --- a/e2e_tests/airlock/request.py +++ b/e2e_tests/airlock/request.py @@ -2,11 +2,12 @@ import logging from httpx import AsyncClient, Timeout import os +import re from urllib.parse import urlparse from azure.storage.blob import BlobClient from azure.storage.blob.aio import BlobClient as AsyncBlobClient -from azure.core.exceptions import ResourceNotFoundError, ResourceExistsError -from uuid import uuid4 +from azure.core.exceptions import HttpResponseError, ResourceNotFoundError, ResourceExistsError +from uuid import UUID, uuid4 from airlock import strings from e2e_tests.helpers import get_auth_header, get_full_endpoint @@ -86,8 +87,30 @@ async def delete_blob_using_sas(file_path: str, sas_url: str): client.delete_blob() -async def wait_for_draft_container_deletion(sas_url: str, *, timeout=60, interval=2): - """Require ContainerNotFound after Azure's asynchronous container deletion.""" +def submitted_container_id(sas_url, subscription_id, tre_id, request_id): + """Bind the ARM observation to this request and the configured v2 account.""" + subscription_id = str(UUID(subscription_id)) + request_id = str(UUID(request_id)) + if not re.fullmatch(r"[a-zA-Z0-9-]+", tre_id): + raise ValueError("Invalid TRE ID for Airlock deletion validation") + account = f"stalairlock{tre_id}".replace("-", "").lower() + parsed = urlparse(sas_url) + if ( + parsed.scheme != "https" + or parsed.netloc != f"{account}.blob.core.windows.net" + or parsed.path != f"/{request_id}-draft" + or not parsed.query + or parsed.fragment + ): + raise ValueError("The draft SAS does not match the test request and configured v2 storage account") + return ( + f"/subscriptions/{subscription_id}/resourceGroups/rg-{tre_id}" + f"/providers/Microsoft.Storage/storageAccounts/{account}/blobServices/default/containers/{request_id}" + ) + + +async def wait_for_draft_container_deletion(sas_url: str, *, arm, container_id, workspace_id, timeout=60, interval=2): + """Require a rejected write and independent ARM evidence of draft deletion.""" started = asyncio.get_running_loop().time() parsed = urlparse(sas_url) # A new name and overwrite avoid mistaking BlobAlreadyExists for revoked access. @@ -101,15 +124,45 @@ async def wait_for_draft_container_deletion(sas_url: str, *, timeout=60, interva except ResourceNotFoundError as error: if getattr(error, "error_code", None) != "ContainerNotFound": raise - LOGGER.info( - "Draft container deletion confirmed after %.1f seconds (ContainerNotFound)", - asyncio.get_running_loop().time() - started, - ) - return + denied = True except ResourceExistsError as error: # Azure can reject writes while deletion is still in progress. if getattr(error, "error_code", None) != "ContainerBeingDeleted": raise + denied = False + except HttpResponseError as error: + # ABAC can deny a SAS operation when deleted container metadata + # is unavailable. This error alone does not prove deletion. + if ( + error.status_code != 403 + or getattr(error, "error_code", None) != "AuthorizationPermissionMismatch" + ): + raise + denied = True + else: + denied = False + if denied: + draft = await arm.request("GET", f"{container_id}-draft", "2023-05-01", missing_ok=True) + if draft is None: + # A live sibling in the same account prevents an incorrect + # subscription, missing account or failed lookup from passing. + submitted = await arm.request("GET", container_id, "2023-05-01") + metadata = submitted.get("properties", {}).get("metadata", {}) + assert submitted.get("id", "").lower() == container_id.lower(), ( + "ARM returned a different submitted container" + ) + assert metadata.get("workspace_id") == workspace_id, ( + "Submitted container belongs to a different workspace" + ) + assert metadata.get("request_type") == "import", "Submitted container is not an import" + assert metadata.get("stage") == "import-in-progress", ( + "Submitted container has an unexpected stage" + ) + LOGGER.info( + "Draft write rejected and ARM deletion confirmed after %.1f seconds", + asyncio.get_running_loop().time() - started, + ) + return await asyncio.sleep(interval) except TimeoutError: raise AssertionError(f"Draft container deletion was not confirmed within {timeout} seconds") from None diff --git a/e2e_tests/test_airlock.py b/e2e_tests/test_airlock.py index d3607bff1d..43cf60d499 100644 --- a/e2e_tests/test_airlock.py +++ b/e2e_tests/test_airlock.py @@ -13,14 +13,17 @@ delete_blob_using_sas, wait_for_status, wait_for_draft_container_deletion, + submitted_container_id, ) from e2e_tests.timeouts import async_test_timeout, cleanup_deadline +from e2e_tests import config from resources.resource import get_resource, post_resource, delete_owned_resource_if_present from resources.workspace import get_workspace_auth_details from airlock import strings as airlock_strings from e2e_tests.conftest import get_workspace_owner_token from helpers import assert_status, get_admin_token, get_auth_header, get_full_endpoint from e2e_tests.token_provider import TokenProvider +from e2e_tests.resources.sql_probe import arm_client pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -96,11 +99,15 @@ async def test_draft_container_is_deleted_after_submit(setup_test_workspace, ver workspace_path, workspace_id = setup_test_workspace workspace_owner_token = await get_workspace_owner_token(workspace_id, verify) - _, container_url = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) + request_id, container_url = await submit_airlock_import_request(workspace_path, workspace_owner_token, verify) # Azure permits draft access for up to 30 seconds after accepting deletion. # The submitted copy is separate. Do not interpret a same-name conflict as denial. - await wait_for_draft_container_deletion(container_url) + container_id = submitted_container_id(container_url, os.environ["ARM_SUBSCRIPTION_ID"], config.TRE_ID, request_id) + async with arm_client() as arm: + await wait_for_draft_container_deletion( + container_url, arm=arm, container_id=container_id, workspace_id=workspace_id + ) async def create_draft_import_request_with_file(workspace_path: str, workspace_owner_token: str, verify: bool): diff --git a/e2e_tests/unit_tests/test_airlock_request.py b/e2e_tests/unit_tests/test_airlock_request.py index 205a42f2f8..a2f6a35a83 100644 --- a/e2e_tests/unit_tests/test_airlock_request.py +++ b/e2e_tests/unit_tests/test_airlock_request.py @@ -1,5 +1,6 @@ """Exercise Airlock response handling without deploying Azure resources.""" +import asyncio import json import unittest from unittest.mock import AsyncMock, MagicMock, patch @@ -62,6 +63,22 @@ def setUp(self): patch.object(request.AsyncBlobClient, "from_blob_url", return_value=self.context) ) self.sas = "https://storage.example.test/draft?sig=private-sas" + self.arm = AsyncMock() + self.container_id = "/subscriptions/test/resourceGroups/rg-test/providers/Microsoft.Storage/storageAccounts/test/blobServices/default/containers/request" + self.submitted = { + "id": self.container_id, + "properties": { + "metadata": {"workspace_id": "workspace", "request_type": "import", "stage": "import-in-progress"} + }, + } + self.arm.request.side_effect = lambda method, resource_id, *args, **kwargs: ( + None if resource_id.endswith("-draft") else self.submitted + ) + + async def wait(self, **kwargs): + await request.wait_for_draft_container_deletion( + self.sas, arm=self.arm, container_id=self.container_id, workspace_id="workspace", **kwargs + ) def missing(self, code="ContainerNotFound"): error = ResourceNotFoundError("Not found") @@ -70,7 +87,7 @@ def missing(self, code="ContainerNotFound"): async def test_transient_writes_are_retried_until_container_not_found(self): self.client.upload_blob.side_effect = [{"etag": "written"}, self.missing()] - await request.wait_for_draft_container_deletion(self.sas, interval=0) + await self.wait(interval=0) self.assertEqual(self.client.upload_blob.await_count, 2) self.assertTrue(self.client.upload_blob.call_args.kwargs["overwrite"]) self.assertIn("/draft/revocation-probe-", self.factory.call_args.args[0]) @@ -79,13 +96,14 @@ async def test_container_being_deleted_is_retried_and_not_accepted_as_complete(s error = ResourceExistsError("Deleting") error.error_code = "ContainerBeingDeleted" self.client.upload_blob.side_effect = [error, self.missing()] - await request.wait_for_draft_container_deletion(self.sas, interval=0) + await self.wait(interval=0) self.assertEqual(self.client.upload_blob.await_count, 2) async def test_writes_that_continue_past_deadline_fail(self): self.client.upload_blob.return_value = {"etag": "still writable"} with self.assertRaisesRegex(AssertionError, "not confirmed within"): - await request.wait_for_draft_container_deletion(self.sas, timeout=0.01, interval=0.001) + await self.wait(timeout=0.01, interval=0.001) + self.arm.request.assert_not_awaited() async def test_conflict_authentication_and_wrong_not_found_errors_fail(self): for error in ( @@ -96,16 +114,103 @@ async def test_conflict_authentication_and_wrong_not_found_errors_fail(self): with self.subTest(error=error): self.client.upload_blob.side_effect = error with self.assertRaises(type(error)): - await request.wait_for_draft_container_deletion(self.sas, interval=0) + await self.wait(interval=0) async def test_probe_names_are_distinct_and_client_closes(self): self.client.upload_blob.side_effect = self.missing() - await request.wait_for_draft_container_deletion(self.sas) + await self.wait() first = self.factory.call_args.args[0] - await request.wait_for_draft_container_deletion(self.sas) + await self.wait() self.assertNotEqual(first, self.factory.call_args.args[0]) self.assertEqual(self.context.__aexit__.await_count, 2) + def forbidden(self, code="AuthorizationPermissionMismatch"): + error = HttpResponseError("Denied") + error.status_code = 403 + error.error_code = code + return error + + async def test_permission_denial_requires_independent_deletion_evidence(self): + self.client.upload_blob.side_effect = self.forbidden() + self.arm.request.side_effect = [{"id": "draft still present"}, None, self.submitted] + await self.wait(interval=0) + self.assertEqual(self.client.upload_blob.await_count, 2) + self.assertEqual(self.arm.request.await_count, 3) + self.arm.request.assert_any_await("GET", f"{self.container_id}-draft", "2023-05-01", missing_ok=True) + self.arm.request.assert_any_await("GET", self.container_id, "2023-05-01") + + async def test_sas_denial_does_not_pass_while_draft_exists(self): + for error in (self.forbidden(), self.missing()): + with self.subTest(error=error): + self.client.upload_blob.side_effect = error + self.arm.request.side_effect = None + self.arm.request.return_value = {"id": "draft still present"} + with self.assertRaisesRegex(AssertionError, "not confirmed within"): + await self.wait(timeout=0.01, interval=0.001) + + async def test_missing_submitted_container_or_arm_access_failure_fails(self): + self.client.upload_blob.side_effect = self.forbidden() + for replies in ([RuntimeError("ARM 403")], [None, RuntimeError("ARM 404")]): + with self.subTest(replies=replies): + self.arm.request.side_effect = replies + with self.assertRaises(RuntimeError): + await self.wait() + + async def test_submitted_container_must_match_request_workspace_and_stage(self): + self.client.upload_blob.side_effect = self.forbidden() + for field in ("id", "workspace_id", "request_type", "stage"): + with self.subTest(field=field): + submitted = json.loads(json.dumps(self.submitted)) + if field == "id": + submitted[field] = "wrong-container" + else: + submitted["properties"]["metadata"][field] = "wrong-value" + self.arm.request.side_effect = [None, submitted] + with self.assertRaises(AssertionError): + await self.wait() + + async def test_other_storage_denials_fail_without_arm_lookup(self): + for code in ("AuthenticationFailed", "AuthorizationFailure", "AuthorizationSourceIPMismatch"): + with self.subTest(code=code): + self.client.upload_blob.side_effect = self.forbidden(code) + with self.assertRaises(HttpResponseError): + await self.wait() + self.arm.request.assert_not_awaited() + + async def test_arm_lookup_shares_the_probe_deadline(self): + self.client.upload_blob.side_effect = self.forbidden() + + async def stalled_lookup(*args, **kwargs): + await asyncio.sleep(60) + + self.arm.request.side_effect = stalled_lookup + with self.assertRaisesRegex(AssertionError, "not confirmed within"): + await self.wait(timeout=0.01) + self.context.__aexit__.assert_awaited_once() + + +class ContainerIdentityTests(unittest.TestCase): + def test_arm_id_is_bound_to_configured_environment_and_request(self): + subscription = "11111111-1111-1111-1111-111111111111" + request_id = "22222222-2222-2222-2222-222222222222" + url = f"https://stalairlocktretest.blob.core.windows.net/{request_id}-draft?sig=private" + result = request.submitted_container_id(url, subscription, "tre-test", request_id) + self.assertEqual( + result, + f"/subscriptions/{subscription}/resourceGroups/rg-tre-test/providers/Microsoft.Storage/storageAccounts/stalairlocktretest/blobServices/default/containers/{request_id}", + ) + for invalid in ( + url.replace("https:", "http:"), + url.replace("tretest.", "other."), + url.replace("-draft?", "?"), + url.replace(request_id, "another-request"), + url.split("?", 1)[0], + url + "#fragment", + ): + with self.subTest(url=invalid): + with self.assertRaises(ValueError): + request.submitted_container_id(invalid, subscription, "tre-test", request_id) + if __name__ == "__main__": unittest.main() From e461a1e4d6a1c33b7915f8dc06c4fbda19a2a386 Mon Sep 17 00:00:00 2001 From: Jay W Date: Tue, 13 Oct 2026 21:43:10 +0100 Subject: [PATCH 09/17] Separate CI and E2E validation from runtime changes (#5119) --- .devcontainer/Dockerfile | 2 +- .../workflows/build_validation_develop.yml | 19 - CHANGELOG.md | 9 +- api_app/_version.py | 2 +- api_app/models/domain/operation.py | 3 - .../service_bus/deployment_status_updater.py | 82 +--- .../test_routes/test_shared_services.py | 44 -- .../test_deployment_status_update.py | 397 ------------------ config_schema.json | 5 +- core/terraform/deploy.sh | 5 +- core/terraform/json-to-env.sh | 50 +-- .../mgmt_storage_private_endpoint.sh | 134 ------ core/terraform/outputs.sh | 88 +--- core/version.txt | 2 +- devops/scripts/terraform_init.sh | 11 +- devops/tests/test_bootstrap.py | 75 ---- devops/tests/test_certificate_cache.py | 311 -------------- devops/tests/test_certificate_script.py | 108 ----- .../test_certificate_storage_readiness.py | 201 --------- devops/tests/test_core_outputs.py | 367 ---------------- .../test_mgmt_storage_private_endpoint.py | 262 ------------ devops/tests/test_review_vm_prerequisites.py | 123 ------ devops/version.txt | 2 +- .../configuring-shared-services.md | 24 -- resource_processor/_version.py | 2 +- resource_processor/tests_rp/test_runner.py | 204 +-------- resource_processor/vmss_porter/runner.py | 68 +-- .../airlock_notifier/Dockerfile.tmpl | 5 - .../airlock_notifier/porter.yaml | 4 +- .../shared_services/certs/Dockerfile.tmpl | 2 +- templates/shared_services/certs/porter.yaml | 10 +- .../certs/scripts/certificate_cache.py | 232 ---------- .../certs/scripts/letsencrypt.sh | 116 +++-- .../certs/scripts/wait_for_private_storage.py | 133 ------ .../certs/terraform/staticweb.tf | 25 +- .../cyclecloud/Dockerfile.tmpl | 5 - .../shared_services/cyclecloud/porter.yaml | 4 +- .../Dockerfile.tmpl | 5 - .../porter.yaml | 12 +- .../Dockerfile.tmpl | 5 - .../porter.yaml | 12 +- .../terraform/windowsvm.tf | 1 - .../guacamole-azure-linuxvm/porter.yaml | 10 +- .../guacamole-azure-windowsvm/porter.yaml | 2 +- .../terraform/vm/locals.tf | 1 - .../terraform/vm/variables.tf | 5 - .../terraform/vm/vm_config.ps1 | 40 +- .../workspace_services/ohdsi/Dockerfile.tmpl | 5 - .../workspace_services/ohdsi/porter.yaml | 4 +- 49 files changed, 169 insertions(+), 3069 deletions(-) delete mode 100755 core/terraform/mgmt_storage_private_endpoint.sh mode change 100755 => 100644 core/terraform/outputs.sh delete mode 100644 devops/tests/test_certificate_cache.py delete mode 100644 devops/tests/test_certificate_script.py delete mode 100644 devops/tests/test_certificate_storage_readiness.py delete mode 100644 devops/tests/test_core_outputs.py delete mode 100644 devops/tests/test_mgmt_storage_private_endpoint.py delete mode 100644 devops/tests/test_review_vm_prerequisites.py delete mode 100644 templates/shared_services/certs/scripts/certificate_cache.py delete mode 100644 templates/shared_services/certs/scripts/wait_for_private_storage.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f39a3b5958..a970d25859 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -43,7 +43,7 @@ ARG DOCKER_CE_VERSION="5:29.1.3-1~debian.12~bookworm" ARG DOCKER_CE_CLI_VERSION="5:29.1.3-1~debian.12~bookworm" ARG DOCKER_CONTAINERD_VERSION="2.2.1-1~debian.12~bookworm" ARG DOCKER_COMPOSE_PLUGIN_VERSION="5.0.0-1~debian.12~bookworm" -RUN apt-get update && apt-get install -y ca-certificates curl dnsutils gnupg lsb-release --no-install-recommends \ +RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release --no-install-recommends \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" \ | tee /etc/apt/sources.list.d/docker.list > /dev/null \ diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index a5f94b9425..3a54f57f12 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -65,12 +65,6 @@ jobs: - '**/porter-build-context.env' ci_cleanup: - 'Makefile' - - 'config_schema.json' - - 'core/terraform/outputs.tf' - - 'core/terraform/outputs.sh' - - 'core/terraform/json-to-env.sh' - - 'devops/scripts/load_env.sh' - - 'devops/scripts/load_and_validate_env.sh' - 'core/terraform/scripts/letsencrypt.sh' - 'core/terraform/scripts/validate_certbot.sh' - 'devops/scripts/clean_ci_validation_envs.sh' @@ -86,8 +80,6 @@ jobs: - 'devops/scripts/destroy_env_no_terraform.sh' - 'devops/scripts/validate_ci_cleanup.py' - 'devops/scripts/kv_add_network_exception.sh' - - 'core/terraform/deploy.sh' - - 'core/terraform/mgmt_storage_private_endpoint.sh' - 'devops/terraform/**' - '.github/actions/devcontainer_run_command/action.yml' - '.github/workflows/deploy_tre_reusable.yml' @@ -95,7 +87,6 @@ jobs: - '.github/workflows/pr_comment_bot.yml' - 'devops/tests/**' - 'templates/shared_services/sonatype-nexus-vm/scripts/**' - - 'templates/shared_services/certs/**' - 'templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/**' - '.github/workflows/build_validation_develop.yml' - '.github/workflows/clean_validation_envs.yml' @@ -140,16 +131,6 @@ jobs: if: ${{ steps.filter.outputs.ci_cleanup == 'true' }} run: python3 -m unittest discover -s devops/tests -v - - name: Test core outputs in the development-container base - if: ${{ steps.filter.outputs.ci_cleanup == 'true' }} - # Bookworm provides jq 1.6; the hosted runner has a newer jq version. - run: | - docker run --rm --network none \ - --mount "type=bind,source=${PWD},target=/workspaces/tre,readonly" \ - --workdir /workspaces/tre --entrypoint /bin/bash \ - mcr.microsoft.com/vscode/devcontainers/python:dev-3.12-bookworm \ - -c 'jq --version; python3 -m unittest discover -s devops/tests -p test_core_outputs.py -v' - - uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 if: ${{ steps.filter.outputs.terraform == 'true' }} with: diff --git a/CHANGELOG.md b/CHANGELOG.md index e58d781a09..6fb5e8dc65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,7 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: -* Validate base-workspace and private Azure SQL lifecycles with bounded cleanup and safe operation-polling diagnostics. ([#5257](https://github.com/microsoft/AzureTRE/pull/5257)) -* Select individual bundle tests, report coverage limits and run selected tests after successful smoke tests. ([#5256](https://github.com/microsoft/AzureTRE/pull/5256)) +* Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL and Airlock storage. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) * Allow numeric CIDR masks in `address_space_size` (e.g. "23") when requesting auto-assigned address spaces; accepts numeric strings and validates the mask range. (`API` 0.28.0, `tre-workspace-base` 2.12.0, `tre-workspace-unrestricted` 0.15.0, `tre-workspace-airlock-import-review` 0.18.0) ([#4733](https://github.com/microsoft/AzureTRE/issues/4733)) @@ -29,11 +28,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: -* Verify Airlock draft-container deletion through independent storage checks and rejected SAS writes. ([#5258](https://github.com/microsoft/AzureTRE/pull/5258)) -* Use supported review VM sizes and validate Windows tools. Forward explicit Nexus consent and bound Airlock and shared-service resource cleanup. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) -* Wait for private certificate storage readiness, reuse trusted certificates and verify TLS for E2E identity authentication. ([#5254](https://github.com/microsoft/AzureTRE/pull/5254)) -* Correct managed-identity login in bundle runtimes. Preserve failed uninstall results, allow cleanup when Porter confirms that an installation is absent, and keep operation status available until the deletion pipeline finishes. ([#5253](https://github.com/microsoft/AzureTRE/pull/5253)) -* Wait for management storage, retry recognised state-read permission failures and validate Terraform outputs. Protect active and recently used validation environments from scheduled cleanup. ([#5252](https://github.com/microsoft/AzureTRE/pull/5252)) +* Protect active validation environments, require successful smoke tests before custom tests, and give E2E cleanup separate deadlines. Forward explicit Nexus consent and verify TLS for identity authentication. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Validate Airlock sort fields against the supported options and avoid returning database error details to callers. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Bind operation retrieval to the authorised resource and return not found for unrelated operations. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/api_app/_version.py b/api_app/_version.py index 2e378084fb..cc09a062c6 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.28.4" +__version__ = "0.28.3" diff --git a/api_app/models/domain/operation.py b/api_app/models/domain/operation.py index 2c37666a26..82342315ed 100644 --- a/api_app/models/domain/operation.py +++ b/api_app/models/domain/operation.py @@ -64,9 +64,6 @@ class OperationStep(AzureTREModel): status: Optional[Status] = Field(None, title="Operation step status") message: Optional[str] = Field("", title="Additional operation step status information") updatedWhen: Optional[float] = Field(default=None, title="POSIX Timestamp for When the operation step was updated") - statusUpdateComplete: bool = Field( - False, description="Whether the terminal status resource writes and next-step dispatch have completed" - ) # An example for this property will be if we have a step that is responsible for updating the firewall, and its origin was the guacamole workspace service, the id here will be the guacamole id sourceTemplateResourceId: Optional[str] = Field( default=None, title="sourceTemplateResourceId", description="Id of the parent of the resource to update" diff --git a/api_app/service_bus/deployment_status_updater.py b/api_app/service_bus/deployment_status_updater.py index 862a2c101c..299dc8a5a8 100644 --- a/api_app/service_bus/deployment_status_updater.py +++ b/api_app/service_bus/deployment_status_updater.py @@ -151,24 +151,7 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage if step_to_update is None: raise f"Error finding step {message.stepId} in operation {message.operationId}" - uninstall_pipeline = operation.action == RequestAction.UnInstall and len(operation.steps) > 1 - if uninstall_pipeline and self.ignore_uninstall_callback(operation, current_step_index, message): - if ( - not step_to_update.statusUpdateComplete - and step_to_update.status == message.status - and operation.status == Status.DeletingFailed - and any(later.is_failure() for later in operation.steps[current_step_index + 1 :]) - ): - # Dispatch can fail after the current step succeeds. Repair - # a missed root write without replaying the earlier effects. - await self.update_primary_uninstall_status(operation) - step_to_update.statusUpdateComplete = True - await self.operations_repo.update_item(operation) - return True - # update the step status - if step_to_update.status != message.status: - step_to_update.statusUpdateComplete = False step_to_update.status = message.status step_to_update.message = message.message step_to_update.updatedWhen = get_timestamp() @@ -183,20 +166,11 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage resource_id = uuid.UUID(step_to_update.resourceId) resource = await self.resource_repo.get_resource_by_id(resource_id) - resource.deploymentStatus = ( - operation.status - if uninstall_pipeline and str(resource_id) == operation.resourceId - else step_to_update.status - ) + resource.deploymentStatus = step_to_update.status await self.resource_repo.update_item(resource) - if uninstall_pipeline and str(resource_id) != operation.resourceId: - await self.update_primary_uninstall_status(operation) # if the step failed, or this queue message is an intermediary ("now deploying..."), return here. if not step_to_update.is_success(): - if uninstall_pipeline and step_to_update.is_failure(): - step_to_update.statusUpdateComplete = True - await self.operations_repo.update_item(operation) return True # update the resource doc to persist any outputs @@ -246,14 +220,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage next_step.status = Status.UpdatingFailed await self.update_overall_operation_status(operation, next_step, is_last_step) await self.operations_repo.update_item(operation) - if uninstall_pipeline: - await self.update_primary_uninstall_status(operation) - - if uninstall_pipeline: - # Save only after the resource writes and dispatch. A retry must - # resume those effects if an earlier database write succeeded. - step_to_update.statusUpdateComplete = True - await self.operations_repo.update_item(operation) result = True @@ -266,32 +232,6 @@ async def update_status_in_database(self, message: DeploymentStatusUpdateMessage return result - def ignore_uninstall_callback(self, operation: Operation, step_index: int, message: DeploymentStatusUpdateMessage): - step = operation.steps[step_index] - if step.is_success() or step.is_failure(): - if step.status != message.status or step.statusUpdateComplete: - logger.info(f"Ignoring completed uninstall callback for {operation.id}/{step.id}") - return True - - # Dispatch follows the previous resource writes. Once a later step has - # progressed, replaying those writes could overwrite newer resource data. - awaiting = (Status.AwaitingDeployment, Status.AwaitingUpdate, Status.AwaitingDeletion, Status.AwaitingAction) - if any(later.status not in awaiting for later in operation.steps[step_index + 1 :]): - logger.info(f"Ignoring earlier uninstall callback for {operation.id}/{step.id}") - return True - - if any(not previous.is_success() for previous in operation.steps[:step_index]): - logger.info(f"Ignoring uninstall callback before its predecessors succeeded: {operation.id}/{step.id}") - return True - return False - - async def update_primary_uninstall_status(self, operation: Operation): - # Cascaded deletion can contain several main steps. The operation owns - # the root resource, which must remain visible until every step finishes. - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(operation.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) - async def update_overall_operation_status(self, operation: Operation, step: OperationStep, is_last_step: bool): operation.updatedWhen = get_timestamp() @@ -308,15 +248,17 @@ async def update_overall_operation_status(self, operation: Operation, step: Oper operation.status = self.get_failure_status_for_action(operation.action) operation.message = f"Multi step pipeline failed on step {step.templateStepId}" - # Uninstall pipelines persist the root status in the caller. Keep - # the existing main-step behaviour for other pipeline actions. - if operation.action != RequestAction.UnInstall: - main_step = next((item for item in operation.steps if item.templateStepId == "main"), None) - if main_step: - primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) - primary_resource.deploymentStatus = operation.status - await self.resource_repo.update_item(primary_resource) - return + # pipeline failed - update the primary resource (from the main step) as failed too + main_step = None + for i, step in enumerate(operation.steps): + if step.templateStepId == "main": + main_step = step + break + + if main_step: + primary_resource = await self.resource_repo.get_resource_by_id(uuid.UUID(main_step.resourceId)) + primary_resource.deploymentStatus = operation.status + await self.resource_repo.update_item(primary_resource) if step.is_success() and is_last_step: operation.status = self.get_success_status_for_action(operation.action) diff --git a/api_app/tests_ma/test_api/test_routes/test_shared_services.py b/api_app/tests_ma/test_api/test_routes/test_shared_services.py index e4fe04d5a2..f168a46416 100644 --- a/api_app/tests_ma/test_api/test_routes/test_shared_services.py +++ b/api_app/tests_ma/test_api/test_routes/test_shared_services.py @@ -1,6 +1,4 @@ import random -import sqlite3 -from contextlib import closing from unittest.mock import AsyncMock import uuid import pytest @@ -14,8 +12,6 @@ from db.errors import EntityDoesNotExist from models.domain.shared_service import SharedService -from models.domain.operation import Status -from db.repositories.shared_services import SharedServiceRepository from resources import strings from auth.rbac import require_tre_admin, require_tre_user_or_admin from azure.cosmos.exceptions import CosmosAccessConditionFailedError @@ -482,43 +478,3 @@ async def test_patch_shared_service_with_invalid_field_returns_422(self, _, app, # Check that the error message contains the key information about the validation error assert "fakeField" in response.text assert "extra inputs are not permitted" in response.text.lower() - - @pytest.mark.parametrize( - "resource_status, expected_http_status", - [(Status.PipelineRunning, 200), (Status.DeletingFailed, 200), (Status.Deleted, 404)], - ) - async def test_uninstall_operation_remains_readable_until_pipeline_finishes( - self, app, client, resource_status, expected_http_status - ): - resource = sample_shared_service() - resource.isEnabled = False - resource.deploymentStatus = resource_status - operation = sample_resource_operation(resource_id=SHARED_SERVICE_ID, operation_id=OPERATION_ID) - operation.action = "uninstall" - operation.status = resource_status - - # Execute the repository's actual equality predicates locally. Only the - # database transport is replaced; the lookup and HTTP dependencies run. - with closing(sqlite3.connect(":memory:")) as database: - database.execute("CREATE TABLE c (id TEXT, resourceType TEXT, deploymentStatus TEXT)") - database.execute("INSERT INTO c VALUES (?, ?, ?)", (resource.id, resource.resourceType, resource_status)) - - def query_documents(query, parameters): - rows = database.execute(query, {item["name"].removeprefix("@"): item["value"] for item in parameters}) - return [resource.model_dump() for _ in rows] - - with ( - patch.object(SharedServiceRepository, "query", side_effect=query_documents), - patch( - "api.dependencies.operations.OperationRepository.get_operation_by_id_for_resource", - return_value=operation, - ) as operation_lookup, - ): - response = await client.get(f"/api/shared-services/{SHARED_SERVICE_ID}/operations/{OPERATION_ID}") - - assert response.status_code == expected_http_status - if expected_http_status == 200: - assert response.json()["operation"]["status"] == resource_status - operation_lookup.assert_awaited_once_with(operation_id=OPERATION_ID, resource_id=SHARED_SERVICE_ID) - else: - operation_lookup.assert_not_awaited() diff --git a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py index a0459cff70..fd215ffa86 100644 --- a/api_app/tests_ma/test_service_bus/test_deployment_status_update.py +++ b/api_app/tests_ma/test_service_bus/test_deployment_status_update.py @@ -11,7 +11,6 @@ from db.errors import EntityDoesNotExist from models.domain.workspace import Workspace -from models.domain.workspace_service import WorkspaceService from models.domain.operation import DeploymentStatusUpdateMessage, Operation, OperationStep, Status from resources import strings from service_bus.deployment_status_updater import DeploymentStatusUpdater @@ -511,90 +510,6 @@ async def test_multi_step_operation_ends_at_last_step( sb_sender_client().get_queue_sender().send_messages.assert_not_called() -@patch("service_bus.deployment_status_updater.ResourceHistoryRepository.create") -@patch("service_bus.deployment_status_updater.ResourceTemplateRepository.create") -@patch("service_bus.deployment_status_updater.update_resource_for_step") -@patch("service_bus.deployment_status_updater.OperationRepository.create") -@patch("service_bus.deployment_status_updater.ResourceRepository.create") -@patch("service_bus.helpers.ServiceBusClient") -@pytest.mark.parametrize("child_status", [Status.Deleted, Status.DeletingFailed]) -async def test_child_uninstall_controls_parent_deletion( - sb_sender_client, - resource_repo, - operations_repo, - update_resource_for_step, - _, - __, - multi_step_operation, - user_resource_multi, - primary_workspace_service_resource, - child_status, -): - operation = copy.deepcopy(multi_step_operation) - operation.id = OPERATION_ID - user_resource_multi.id = "0e569b65-7e01-4a0b-a971-a41f5110c0c1" - primary_workspace_service_resource.id = "dc9b2f53-0784-4c4f-9b73-4a060a385bca" - parent = WorkspaceService(**primary_workspace_service_resource.model_dump(), workspaceId="test-workspace") - parent.isEnabled = False - parent.deploymentStatus = Status.Updated - operation.action = RequestAction.UnInstall - operation.steps = operation.steps[:2] - child_step, parent_step = operation.steps - child_step.resourceId = user_resource_multi.id - child_step.resourceAction = RequestAction.UnInstall - child_step.status = Status.Deleting - parent_step.resourceId = primary_workspace_service_resource.id - parent_step.resourceAction = RequestAction.UnInstall - parent_step.status = Status.AwaitingDeletion - operation.resourceId = primary_workspace_service_resource.id - operation.status = Status.PipelineRunning - for step, resource in [(child_step, user_resource_multi), (parent_step, parent)]: - step.templateStepId = "main" - step.sourceTemplateResourceId = resource.id - step.resourceTemplateName = resource.templateName - step.resourceType = resource.resourceType - parent_before = parent_step.model_copy(deep=True) - - message = { - "operationId": operation.id, - "stepId": child_step.id, - "id": child_step.resourceId, - "status": child_status, - "message": "Child uninstall result", - } - operations_repo.return_value.get_operation_by_id.return_value = operation - resources = { - user_resource_multi.id: user_resource_multi, - parent.id: parent, - } - resource_repo.return_value.get_resource_by_id.side_effect = lambda resource_id: resources[str(resource_id)] - resource_repo.return_value.get_resource_dict_by_id.side_effect = lambda resource_id: resources[ - str(resource_id) - ].model_dump() - update_resource_for_step.return_value = parent - sb_sender_client().get_queue_sender().send_messages = AsyncMock() - - updater = DeploymentStatusUpdater() - await updater.init_repos() - complete = await updater.process_message(ServiceBusReceivedMessageMock(message)) - - assert complete is True - saved_operation = operations_repo.return_value.update_item.call_args.args[0] - expected_parent_status = Status.DeletingFailed if child_status == Status.DeletingFailed else Status.PipelineRunning - assert saved_operation.status == expected_parent_status - assert saved_operation.steps[0].status == child_status - assert saved_operation.steps[0].message == message["message"] - assert saved_operation.steps[1] == parent_before - assert user_resource_multi.deploymentStatus == child_status - assert parent.deploymentStatus == expected_parent_status - if child_status == Status.DeletingFailed: - update_resource_for_step.assert_not_called() - sb_sender_client().get_queue_sender().send_messages.assert_not_called() - else: - update_resource_for_step.assert_called_once() - sb_sender_client().get_queue_sender().send_messages.assert_called_once() - - async def test_convert_outputs_to_dict(): # Test case 1: Empty list of outputs outputs_list = [] @@ -618,315 +533,3 @@ async def test_convert_outputs_to_dict(): "list2": ["one", "two"], } assert status_updater.convert_outputs_to_dict(deployment_status_update_message.outputs) == expected_result - - -@pytest.mark.parametrize( - "phase, message_status, expected_operation_status, expected_primary_status", - [ - ("main", Status.Deleted, Status.PipelineRunning, Status.PipelineRunning), - ("final", Status.Updated, Status.Deleted, Status.Deleted), - ("final", Status.UpdatingFailed, Status.DeletingFailed, Status.DeletingFailed), - ("enqueue_failed", Status.Deleted, Status.DeletingFailed, Status.DeletingFailed), - ], -) -async def test_uninstall_waits_for_final_pipeline_step( - phase, message_status, expected_operation_status, expected_primary_status -): - primary_id = "d9236f8f-7fbb-46d3-9c94-dea6daf32b7f" - dependency_id = "7af8f146-d751-4587-a22e-eb44c1c6fbe2" - primary = create_sample_workspace_object(primary_id) - primary.isEnabled = False - primary.deploymentStatus = Status.PipelineRunning if phase == "final" else Status.Deleting - dependency = create_sample_workspace_object(dependency_id) - dependency.deploymentStatus = Status.Updating - stored = {primary_id: primary, dependency_id: dependency} - - operation = create_sample_operation(primary_id, RequestAction.UnInstall) - operation.steps[0].status = Status.Deleted if phase == "final" else Status.Deleting - operation.steps.append( - OperationStep( - id="firewall-step", - templateStepId="remove-firewall-rules", - resourceId=dependency_id, - resourceAction=RequestAction.Upgrade, - sourceTemplateResourceId=primary_id, - status=Status.Updating if phase == "final" else Status.AwaitingUpdate, - ) - ) - operation.status = Status.PipelineRunning - current_step = operation.steps[1 if phase == "final" else 0] - message = DeploymentStatusUpdateMessage( - operationId=operation.id, - stepId=current_step.id, - id=current_step.resourceId, - status=message_status, - message="Final firewall update failed" if message_status == Status.UpdatingFailed else "Completed", - ) - - updater = DeploymentStatusUpdater() - updater.operations_repo = AsyncMock() - updater.operations_repo.get_operation_by_id.return_value = operation - updater.resource_repo = AsyncMock() - updater.resource_template_repo = AsyncMock() - updater.resource_history_repo = AsyncMock() - updater.resource_repo.get_resource_by_id.side_effect = lambda resource_id: stored[str(resource_id)].model_copy( - deep=True - ) - updater.resource_repo.get_resource_dict_by_id.side_effect = lambda resource_id: stored[ - str(resource_id) - ].model_dump() - - def save_resource(resource): - stored[resource.id] = resource.model_copy(deep=True) - - def save_resource_dict(resource): - stored[resource["id"]] = stored[resource["id"]].model_copy(update=resource, deep=True) - - updater.resource_repo.update_item.side_effect = save_resource - updater.resource_repo.update_item_dict.side_effect = save_resource_dict - with ( - patch("service_bus.deployment_status_updater.update_resource_for_step", return_value=dependency), - patch("service_bus.deployment_status_updater.send_deployment_message", new_callable=AsyncMock) as send, - ): - if phase == "enqueue_failed": - send.side_effect = RuntimeError("Cannot enqueue final step") - assert await updater.update_status_in_database(message) is True - - assert operation.status == expected_operation_status - assert stored[primary_id].deploymentStatus == expected_primary_status - assert stored[primary_id].isEnabled is False - if phase != "final": - assert stored[primary_id].deploymentStatus != Status.Deleted - send.assert_awaited_once() - else: - assert stored[dependency_id].deploymentStatus == message_status - send.assert_not_awaited() - if message_status == Status.UpdatingFailed: - assert operation.message == "Multi step pipeline failed on step remove-firewall-rules" - assert operation.steps[-1].message == "Final firewall update failed" - if phase == "enqueue_failed": - assert operation.steps[-1].status == Status.UpdatingFailed - assert "Cannot enqueue final step" in operation.steps[-1].message - - -@pytest.fixture -def persisted_uninstall_pipeline(): - primary = create_sample_workspace_object("d9236f8f-7fbb-46d3-9c94-dea6daf32b7f") - primary.isEnabled = False - primary.deploymentStatus = Status.Deleting - dependency = create_sample_workspace_object("7af8f146-d751-4587-a22e-eb44c1c6fbe2") - dependency.deploymentStatus = Status.Updated - operation = create_sample_operation(primary.id, RequestAction.UnInstall) - operation.status = Status.PipelineRunning - operation.steps[0].status = Status.Deleting - operation.steps.append( - OperationStep( - id="firewall-step", - templateStepId="remove-firewall-rules", - resourceId=dependency.id, - resourceAction=RequestAction.Upgrade, - sourceTemplateResourceId=primary.id, - status=Status.AwaitingUpdate, - ) - ) - stored = {"operation": operation, "resources": {primary.id: primary, dependency.id: dependency}} - updater = DeploymentStatusUpdater() - updater.operations_repo = AsyncMock() - updater.resource_repo = AsyncMock() - updater.resource_template_repo = AsyncMock() - updater.resource_history_repo = AsyncMock() - updater.operations_repo.get_operation_by_id.side_effect = lambda _: stored["operation"].model_copy(deep=True) - - def save_operation(item): - stored["operation"] = item.model_copy(deep=True) - - def save_resource(item): - stored["resources"][item.id] = item.model_copy(deep=True) - - def save_resource_dict(item): - original = stored["resources"][item["id"]] - stored["resources"][item["id"]] = original.model_copy(update=item, deep=True) - - updater.operations_repo.update_item.side_effect = save_operation - updater.resource_repo.get_resource_by_id.side_effect = lambda item_id: stored["resources"][str(item_id)].model_copy( - deep=True - ) - updater.resource_repo.get_resource_dict_by_id.side_effect = lambda item_id: stored["resources"][ - str(item_id) - ].model_dump() - updater.resource_repo.update_item.side_effect = save_resource - updater.resource_repo.update_item_dict.side_effect = save_resource_dict - - def message(step_index, status): - step = stored["operation"].steps[step_index] - return DeploymentStatusUpdateMessage( - operationId=operation.id, stepId=step.id, id=step.resourceId, status=status, message="Result" - ) - - with ( - patch("service_bus.deployment_status_updater.update_resource_for_step", return_value=dependency), - patch("service_bus.deployment_status_updater.send_deployment_message", new_callable=AsyncMock) as send, - ): - yield updater, stored, message, send - - -async def test_uninstall_redelivery_does_not_dispatch_next_step_twice(persisted_uninstall_pipeline): - updater, stored, message, send = persisted_uninstall_pipeline - callback = message(0, Status.Deleted) - assert await updater.update_status_in_database(callback) is True - assert stored["operation"].steps[0].statusUpdateComplete is True - before = copy.deepcopy(stored) - - assert await updater.update_status_in_database(callback) is True - - assert stored == before - send.assert_awaited_once() - - -@pytest.mark.parametrize("final_status", [Status.Updated, Status.UpdatingFailed]) -@pytest.mark.parametrize( - "callback_index, callback_status", - [(0, Status.Deleted), (0, Status.Deleting), (1, Status.Updating), (1, Status.Updated)], -) -async def test_uninstall_redelivery_preserves_final_outcome( - persisted_uninstall_pipeline, final_status, callback_index, callback_status -): - updater, stored, message, send = persisted_uninstall_pipeline - assert await updater.update_status_in_database(message(0, Status.Deleted)) is True - assert await updater.update_status_in_database(message(1, final_status)) is True - before = copy.deepcopy(stored) - - assert await updater.update_status_in_database(message(callback_index, callback_status)) is True - - assert stored == before - send.assert_awaited_once() - - -@pytest.mark.parametrize("later_status", [Status.Updating, Status.Updated, Status.UpdatingFailed]) -async def test_uninstall_older_callback_does_not_repeat_writes_after_later_progress( - persisted_uninstall_pipeline, later_status -): - updater, stored, message, send = persisted_uninstall_pipeline - # Older operations have no checkpoint. A later callback also proves dispatch - # completed if the process stopped before it could save the checkpoint. - operation = stored["operation"] - operation.steps[0].status = Status.Deleted - operation.steps[1].status = later_status - operation.status = { - Status.Updating: Status.PipelineRunning, - Status.Updated: Status.Deleted, - Status.UpdatingFailed: Status.DeletingFailed, - }[later_status] - legacy = operation.model_dump() - for step in legacy["steps"]: - del step["statusUpdateComplete"] - stored["operation"] = Operation.model_validate(legacy) - before = copy.deepcopy(stored) - - assert await updater.update_status_in_database(message(0, Status.Deleted)) is True - - if later_status == Status.UpdatingFailed: - # A legacy record can also be an interrupted dispatch-failure write. - # Only the saved root outcome and the earlier checkpoint need repair. - before["operation"].steps[0].statusUpdateComplete = True - before["resources"][operation.resourceId].deploymentStatus = Status.DeletingFailed - updater.resource_repo.update_item.assert_awaited_once_with(before["resources"][operation.resourceId]) - else: - updater.resource_repo.update_item.assert_not_awaited() - assert stored == before - updater.resource_repo.update_item_dict.assert_not_awaited() - send.assert_not_awaited() - - -@pytest.mark.parametrize("write_method", ["update_item", "update_item_dict"]) -async def test_uninstall_redelivery_resumes_after_partial_resource_write(persisted_uninstall_pipeline, write_method): - updater, stored, message, send = persisted_uninstall_pipeline - writer = getattr(updater.resource_repo, write_method) - save = writer.side_effect - writer.side_effect = RuntimeError("Interrupted database write") - callback = message(0, Status.Deleted) - assert await updater.update_status_in_database(callback) is False - assert stored["operation"].steps[0].status == Status.Deleted - assert stored["operation"].steps[0].statusUpdateComplete is False - send.assert_not_awaited() - - writer.side_effect = save - assert await updater.update_status_in_database(callback) is True - - assert stored["operation"].steps[0].statusUpdateComplete is True - assert stored["resources"][str(callback.id)].deploymentStatus == Status.PipelineRunning - send.assert_awaited_once() - - -@pytest.mark.parametrize("final_status", [Status.Updated, Status.UpdatingFailed]) -async def test_uninstall_redelivery_finishes_terminal_resource_write(persisted_uninstall_pipeline, final_status): - updater, stored, message, send = persisted_uninstall_pipeline - assert await updater.update_status_in_database(message(0, Status.Deleted)) is True - writer = updater.resource_repo.update_item - save = writer.side_effect - - def fail_primary_write(item): - if item.id == stored["operation"].resourceId: - raise RuntimeError("Interrupted primary resource write") - save(item) - - writer.side_effect = fail_primary_write - callback = message(1, final_status) - assert await updater.update_status_in_database(callback) is False - assert stored["operation"].steps[1].statusUpdateComplete is False - - writer.side_effect = save - assert await updater.update_status_in_database(callback) is True - - expected = Status.Deleted if final_status == Status.Updated else Status.DeletingFailed - assert stored["operation"].status == expected - assert stored["operation"].steps[1].statusUpdateComplete is True - assert stored["resources"][stored["operation"].resourceId].deploymentStatus == expected - send.assert_awaited_once() - - -async def test_uninstall_checkpoint_failure_does_not_report_dispatch_failure(persisted_uninstall_pipeline): - updater, stored, message, send = persisted_uninstall_pipeline - writer = updater.operations_repo.update_item - save = writer.side_effect - - def fail_checkpoint(item): - if item.steps[0].statusUpdateComplete: - raise RuntimeError("Interrupted checkpoint write") - save(item) - - writer.side_effect = fail_checkpoint - assert await updater.update_status_in_database(message(0, Status.Deleted)) is False - - assert stored["operation"].status == Status.PipelineRunning - assert stored["operation"].steps[1].status == Status.AwaitingUpdate - assert stored["operation"].steps[0].statusUpdateComplete is False - send.assert_awaited_once() - - -async def test_uninstall_redelivery_repairs_root_after_dispatch_failure(persisted_uninstall_pipeline): - updater, stored, message, send = persisted_uninstall_pipeline - send.side_effect = RuntimeError("Cannot enqueue final step") - writer = updater.resource_repo.update_item - save = writer.side_effect - - def fail_failure_write(item): - if item.deploymentStatus == Status.DeletingFailed: - raise RuntimeError("Interrupted root failure write") - save(item) - - writer.side_effect = fail_failure_write - callback = message(0, Status.Deleted) - assert await updater.update_status_in_database(callback) is False - assert stored["operation"].status == Status.DeletingFailed - assert stored["resources"][str(callback.id)].deploymentStatus == Status.PipelineRunning - assert stored["operation"].steps[0].statusUpdateComplete is False - updater.resource_repo.update_item_dict.reset_mock() - - writer.side_effect = save - assert await updater.update_status_in_database(callback) is True - - assert stored["resources"][str(callback.id)].deploymentStatus == Status.DeletingFailed - assert stored["operation"].steps[0].statusUpdateComplete is True - updater.resource_repo.update_item_dict.assert_not_awaited() - send.assert_awaited_once() diff --git a/config_schema.json b/config_schema.json index 82a042aa3d..56eba4a703 100644 --- a/config_schema.json +++ b/config_schema.json @@ -219,10 +219,7 @@ "workspace_api_client_secret": { "description": "Workspace AD Application secret. This will be created for you for future use - when creating workspaces.", "type": "string", - "minLength": 11, - "not": { - "pattern": "['\\r\\n\\u0000]" - } + "minLength": 11 } }, "required": [ diff --git a/core/terraform/deploy.sh b/core/terraform/deploy.sh index a954bbf726..f24107f32e 100755 --- a/core/terraform/deploy.sh +++ b/core/terraform/deploy.sh @@ -27,8 +27,6 @@ LOG_FILE="${TS}-tre-core.log" # core deployments. This may increase the risk of hitting Azure API rate limits. # If deployment reliability issues arise (e.g., frequent throttling), consider # lowering the parallelism value. -# The management storage private endpoint is created before the main apply because -# creating it changes the DNS of the storage account that holds the Terraform state. # This variables are loaded in for us # shellcheck disable=SC2154 ../../devops/scripts/terraform_wrapper.sh \ @@ -38,8 +36,7 @@ LOG_FILE="${TS}-tre-core.log" -n "${TF_VAR_terraform_state_container_name}" \ -k "${TRE_ID}" \ -l "${LOG_FILE}" \ - -c "./mgmt_storage_private_endpoint.sh && \ - terraform plan --parallelism=25 -out ${PLAN_FILE} && \ + -c "terraform plan --parallelism=25 -out ${PLAN_FILE} && \ terraform apply -input=false -auto-approve --parallelism=25 ${PLAN_FILE} && \ terraform output -json > ../tre_output.json" diff --git a/core/terraform/json-to-env.sh b/core/terraform/json-to-env.sh index a3d824b013..0360fc2cc4 100755 --- a/core/terraform/json-to-env.sh +++ b/core/terraform/json-to-env.sh @@ -1,16 +1,9 @@ #!/bin/bash set -e -set -o pipefail echo "# Generated environment variables from tf output" -jq -ers ' - if length != 1 or (.[0] | type) != "object" then - error("Expected one Terraform output object. Check the state and regenerate core/tre_output.json.") - else - .[0] - end - | +jq -r ' [ { "path": "core_resource_group_name", @@ -122,40 +115,25 @@ jq -ers ' "env_var": "EVENT_GRID_AIRLOCK_NOTIFICATION_TOPIC_RESOURCE_ID" }, { - # This mapping is retained for older caches but is not a root output. "path": "airlock_malware_scan_result_topic_name", - "env_var": "AIRLOCK_MALWARE_SCAN_RESULT_TOPIC_NAME", - "optional": true + "env_var": "AIRLOCK_MALWARE_SCAN_RESULT_TOPIC_NAME" } ] as $env_vars_to_extract | - . as $outputs + with_entries( + select ( + .key as $a + | + any( $env_vars_to_extract[]; .path == $a) + ) + | + .key |= . as $old_key | ($env_vars_to_extract[] | select (.path == $old_key) | .env_var) + ) | - $env_vars_to_extract + to_entries | - map( - . as $mapping - | if $mapping.optional and ($outputs | has($mapping.path) | not) then - empty - else - $outputs[$mapping.path] - | if type != "object" then - error("Missing or invalid Terraform output: \($mapping.path). Check the state and regenerate core/tre_output.json.") - elif (.value | type) != "string" then - error("Terraform output \($mapping.path) must have a string value. Check the state and regenerate core/tre_output.json.") - elif (.value | test("\\S") | not) then - error("Terraform output \($mapping.path) is empty. Check the state and regenerate core/tre_output.json.") - # jq 1.6 treats a NUL needle as empty in contains(). - elif (.value | index("\u0000") != null or test("[\r\n\u0027]")) then - error("Terraform output \($mapping.path) contains a single quote, newline or NUL. Check the state and regenerate core/tre_output.json.") - else - # load_env.sh strips outer quotes without interpreting shell escapes. - # load_and_validate_env.sh also sources this file, so reject single quotes above. - "\($mapping.env_var)=\u0027\(.value)\u0027" - end - end - ) + map("\(.key)=\"\(.value.value)\"") | .[] - ' + ' | sed "s/\"/'/g" # replace double quote with single quote to handle special chars diff --git a/core/terraform/mgmt_storage_private_endpoint.sh b/core/terraform/mgmt_storage_private_endpoint.sh deleted file mode 100755 index 90a2d01707..0000000000 --- a/core/terraform/mgmt_storage_private_endpoint.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# -# Reconcile the resource processor's management storage private endpoint before the main core apply. -# -# The management storage account also holds the Terraform state. Creating its private endpoint -# changes the storage account's public DNS alias chain, so state reads and writes during the same -# apply can fail with "no such host". This script reconciles the private endpoint in a targeted -# apply, then verifies the new DNS alias and backend reads before the main apply starts. -# -# Run from core/terraform after "terraform init". -# - -set -o errexit -set -o pipefail -set -o nounset - -PE_ADDRESS='module.resource_processor_vmss_porter[0].azurerm_private_endpoint.mgmtblobpe' -DNS_REQUIRED_SUCCESSES="${MGMT_STORAGE_DNS_REQUIRED_SUCCESSES:-3}" -DNS_MAX_ATTEMPTS="${MGMT_STORAGE_DNS_MAX_ATTEMPTS:-30}" -DNS_INTERVAL_SECONDS="${MGMT_STORAGE_DNS_INTERVAL_SECONDS:-10}" - -function timestamp() { - date -u +"%Y-%m-%dT%H:%M:%SZ" -} - -function state_blob_host() { - local endpoint - # shellcheck disable=SC2154 - endpoint=$(az storage account show \ - --name "${TF_VAR_mgmt_storage_account_name}" \ - --resource-group "${TF_VAR_mgmt_resource_group_name}" \ - --query primaryEndpoints.blob --output tsv) - endpoint="${endpoint#https://}" - echo "${endpoint%%/*}" -} - -function state_blob_is_reachable() { - local exists - # Check the data plane without initialising state or acquiring a state lease. - # An authenticated not-found response is sufficient for a new state blob. - # shellcheck disable=SC2154 - if ! exists=$(timeout 30 az storage blob exists \ - --account-name "${TF_VAR_mgmt_storage_account_name}" \ - --container-name "${TF_VAR_terraform_state_container_name}" \ - --name "${TRE_ID}" --auth-mode login --only-show-errors \ - --query exists --output tsv); then - return 1 - fi - case "${exists}" in - true|false) return 0 ;; - *) echo "Unexpected response from the state blob existence check" >&2; return 1 ;; - esac -} - -function wait_for_stable_dns() { - local host="$1" - local expected_alias="${host%%.*}.privatelink.${host#*.}" - local successes=0 - local attempt aliases answer backend_error - - echo "Waiting for ${expected_alias}, address resolution and backend access (${DNS_REQUIRED_SUCCESSES} consecutive checks)" - for ((attempt=1; attempt<=DNS_MAX_ATTEMPTS; attempt++)); do - aliases=$(dig +time=5 +tries=1 +short CNAME "${host}" 2>&1) || aliases="" - if ! grep -ixFq "${expected_alias}." <<< "${aliases}"; then - successes=0 - echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: expected alias not observed: ${aliases:-no answer}" - elif ! answer=$(timeout 15 getent ahosts "${host}" 2>&1) || [[ -z "${answer}" ]]; then - successes=0 - echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: ${host} did not resolve ${answer}" - elif ! backend_error=$(state_blob_is_reachable 2>&1); then - successes=0 - echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: backend read failed: ${backend_error}" - else - successes=$((successes + 1)) - echo "$(timestamp) DNS attempt ${attempt}/${DNS_MAX_ATTEMPTS}: alias, address and backend ready (${successes}/${DNS_REQUIRED_SUCCESSES})" - echo " CNAME: ${aliases}" - echo "${answer}" | awk '{print " " $0}' - if [[ "${successes}" -ge "${DNS_REQUIRED_SUCCESSES}" ]]; then - return 0 - fi - fi - if [[ "${attempt}" -lt "${DNS_MAX_ATTEMPTS}" ]]; then - sleep "${DNS_INTERVAL_SECONDS}" - fi - done - - echo "Error: ${host} did not become ready after ${DNS_MAX_ATTEMPTS} attempts." >&2 - echo "Configured resolver diagnostics:" - dig +time=2 +tries=1 +noall +answer +comments +stats "${host}" A || true - echo "Azure resolver diagnostics (168.63.129.16):" - dig @168.63.129.16 +time=2 +tries=1 +noall +answer +comments +stats "${host}" A || true - echo "The main apply has not started. Inspect DNS and backend access, then re-run to repeat these checks." >&2 - return 1 -} - -if [[ "${TF_VAR_resource_processor_type:-vmss_porter}" != "vmss_porter" ]]; then - echo "Resource processor type is not vmss_porter; skipping management storage private endpoint step" - exit 0 -fi - -for required_command in dig getent timeout; do - if ! command -v "${required_command}" >/dev/null; then - echo "Error: ${required_command} is required. Rebuild the repository development container." >&2 - exit 1 - fi -done - -host=$(state_blob_host) -echo "$(timestamp) Planning management storage private endpoint before the main core apply" -timeout 15 getent ahosts "${host}" | awk '{print " before: " $0}' || echo " before: ${host} did not resolve" - -# State presence alone misses tainted resources, replacement and remote deletion. -# Refresh the target and its dependencies, and apply the saved plan only when needed. -PE_PLAN_FILE="$(date +"%s")-tre-core-mgmt-pe.tfplan" -if terraform plan -input=false -detailed-exitcode -target="${PE_ADDRESS}" -out "${PE_PLAN_FILE}"; then - plan_exit=0 -else - plan_exit=$? -fi -case "${plan_exit}" in - 0) - echo "Management storage private endpoint target has no planned changes; skipping targeted apply" - ;; - 2) - terraform apply -input=false -auto-approve "${PE_PLAN_FILE}" - echo "$(timestamp) Management storage private endpoint apply finished" - ;; - *) - echo "Error: management storage private endpoint plan failed (exit ${plan_exit}). The main apply has not started." >&2 - exit "${plan_exit}" - ;; -esac - -wait_for_stable_dns "${host}" diff --git a/core/terraform/outputs.sh b/core/terraform/outputs.sh old mode 100755 new mode 100644 index 5538105986..4b2ffa8347 --- a/core/terraform/outputs.sh +++ b/core/terraform/outputs.sh @@ -1,93 +1,41 @@ #!/bin/bash set -e -# This file is sourced inside Makefile && chains, where Bash ignores errexit. -# Check failures explicitly so callers cannot continue with missing outputs. # shellcheck disable=SC1091 # shellcheck disable=SC2154 source ../../devops/scripts/storage_enable_public_access.sh \ --storage-account-name "${TF_VAR_mgmt_storage_account_name}" \ - --resource-group-name "${TF_VAR_mgmt_resource_group_name}" || return $? + --resource-group-name "${TF_VAR_mgmt_resource_group_name}" -core_outputs_file=../tre_output.json -core_outputs_temporary="" -if [ ! -s "$core_outputs_file" ]; then - # Connect to the remote backend of Terraform. +if [ ! -f ../tre_output.json ] || [ ! -s ../tre_output.json ]; then + # Connect to the remote backend of Terraform export TF_LOG="" # shellcheck disable=SC2154 terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=$TF_VAR_mgmt_resource_group_name" \ -backend-config="storage_account_name=$TF_VAR_mgmt_storage_account_name" \ -backend-config="container_name=$TF_VAR_terraform_state_container_name" \ - -backend-config="key=${TRE_ID}" || return $? + -backend-config="key=${TRE_ID}" - # Keep Terraform output temporary until it has been validated and converted. - core_outputs_temporary=$(mktemp ../tre_output.json.tmp.XXXXXX) || return $? - if terraform output -json > "$core_outputs_temporary"; then - core_outputs_file="$core_outputs_temporary" - else - core_outputs_exit=$? - rm -f "$core_outputs_temporary" - return "$core_outputs_exit" - fi + # Convert the output to json + terraform output -json > ../tre_output.json fi -if ! jq -e 'type == "object" and length > 0' "$core_outputs_file" > /dev/null; then - if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi - echo "Terraform outputs are empty or invalid. Check the state and regenerate core/tre_output.json." >&2 - return 1 -fi +# Now create an .env file +./json-to-env.sh < ../tre_output.json > ../private.env -# Pull in the core template environment variables for the additional values. +# Pull in the core templates environment variables so we can build up new key/value pairs if [ -f ../.env ]; then # shellcheck disable=SC1091 - source ../.env || { - core_outputs_exit=$? - if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi - return "$core_outputs_exit" - } + source ../.env fi -# Both environment readers require literal, single-line, single-quoted values. -# Bash variables cannot contain NUL bytes. -for core_env_variable in WORKSPACE_API_CLIENT_ID WORKSPACE_API_CLIENT_SECRET SUB_ID TENANT_ID; do - case "${!core_env_variable}" in - *"'"*|*$'\r'*|*$'\n'*) - if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi - echo "${core_env_variable} contains a single quote or newline. Correct the value and regenerate core/private.env." >&2 - return 1 - ;; - esac -done +# Add a few extra values to the file to help us (i.e. for local debugging api_app and resource processor) +# shellcheck disable=SC2129 +echo "TEST_WORKSPACE_APP_ID='${WORKSPACE_API_CLIENT_ID}'" >> ../private.env +echo "TEST_WORKSPACE_APP_SECRET='${WORKSPACE_API_CLIENT_SECRET}'" >> ../private.env -# Preserve the previous environment file if conversion or writing fails. -core_env_temporary=$(mktemp ../private.env.tmp.XXXXXX) || { - core_outputs_exit=$? - if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi - return "$core_outputs_exit" -} -if ./json-to-env.sh < "$core_outputs_file" > "$core_env_temporary" && \ - printf "%s='%s'\n" \ - TEST_WORKSPACE_APP_ID "${WORKSPACE_API_CLIENT_ID}" \ - TEST_WORKSPACE_APP_SECRET "${WORKSPACE_API_CLIENT_SECRET}" \ - SUBSCRIPTION_ID "${SUB_ID}" \ - AZURE_SUBSCRIPTION_ID "${SUB_ID}" \ - AZURE_TENANT_ID "${TENANT_ID}" >> "$core_env_temporary"; then - if [ -n "$core_outputs_temporary" ]; then - mv "$core_outputs_temporary" ../tre_output.json || { - core_outputs_exit=$? - rm -f "$core_outputs_temporary" "$core_env_temporary" - return "$core_outputs_exit" - } - fi - mv "$core_env_temporary" ../private.env || { - core_outputs_exit=$? - rm -f "$core_env_temporary" - return "$core_outputs_exit" - } -else - core_outputs_exit=$? - if [ -n "$core_outputs_temporary" ]; then rm -f "$core_outputs_temporary"; fi - rm -f "$core_env_temporary" - return "$core_outputs_exit" -fi +# These next ones from Check Dependencies +echo "SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env +echo "AZURE_SUBSCRIPTION_ID='${SUB_ID}'" >> ../private.env +echo "AZURE_TENANT_ID='${TENANT_ID}'" >> ../private.env diff --git a/core/version.txt b/core/version.txt index 0639f54b2f..2cfeb05209 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.18.13" +__version__ = "0.18.11" diff --git a/devops/scripts/terraform_init.sh b/devops/scripts/terraform_init.sh index 642980b8d8..096b1d65d0 100755 --- a/devops/scripts/terraform_init.sh +++ b/devops/scripts/terraform_init.sh @@ -1,7 +1,7 @@ #!/bin/bash # Source this helper before initialising a management Terraform backend. -# Retry recognised probe, workspace-listing and state-read permission failures. +# Retry recognised probe permissions and workspace-listing failures before state locking. retry_with_backoff() { local func="$1" @@ -136,14 +136,7 @@ init_terraform() { echo "ERROR: Check the reported lock owner before attempting state recovery. No lock has been released by this helper." >&2 return 2 fi - # Terraform's Azure backend returns state-read errors before locking or after - # releasing its lease. Lock and unlock failures must pass the guard above. - # Match the known storage code, not an unrelated 403 or unknown state error. - if grep -Fq 'Error loading state:' <<< "$terraform_output" \ - && grep -Fq 'AuthorizationPermissionMismatch' <<< "$terraform_output"; then - return 1 - fi - # Workspace listing also occurs before state locking. + # Retry only workspace-listing permission failures, not unknown state errors. if grep -Fq 'Failed to get existing workspaces' <<< "$terraform_output" \ && is_storage_permission_error "$terraform_output"; then return 1 diff --git a/devops/tests/test_bootstrap.py b/devops/tests/test_bootstrap.py index 98ce2639e3..0182f7b547 100644 --- a/devops/tests/test_bootstrap.py +++ b/devops/tests/test_bootstrap.py @@ -31,14 +31,6 @@ If you want to change the default action to apply when no rule matches, please use 'az storage account update'. """ -# Diagnostic from fresh management deployment 38181301145, attempt 1. -STATE_READ_PERMISSION_ERROR = ( - "Error: Error loading state: executing request: unexpected status 403 " - "(403 This request is not authorized to perform this operation using this permission.) " - "with AuthorizationPermissionMismatch: This request is not authorized to perform " - "this operation using this permission." -) - MOCK_COMMAND = r""" import hashlib import json @@ -786,44 +778,6 @@ def test_unknown_init_permission_error_is_not_blindly_retried(self): self.assertIn("Error loading state: HTTP 403", self.output) self.assert_no_imports() - def test_state_read_permission_failure_after_probe_is_retried(self): - result = self.run_bootstrap( - init_responses=[ - {"code": 1, "output": STATE_READ_PERMISSION_ERROR}, - {"code": 0, "output": "Backend ready"}, - ] - ) - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(len(self.commands("terraform", "init")), 2) - self.assertEqual(self.sleeps(), [10]) - self.assertIn(STATE_READ_PERMISSION_ERROR, self.output) - - def test_state_read_permission_retries_are_bounded(self): - result = self.run_bootstrap(init_responses=[{"code": 1, "output": STATE_READ_PERMISSION_ERROR}]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.commands("terraform", "init")), 6) - self.assertEqual(self.sleeps(), [10, 20, 40, 80, 160]) - self.assertEqual(self.output.count(STATE_READ_PERMISSION_ERROR), 6) - self.assert_no_imports() - - def test_state_read_permission_error_does_not_hide_lock_failure(self): - error = f"{STATE_READ_PERMISSION_ERROR}\nfailed to lock azure state: failed to write lock metadata" - result = self.run_bootstrap(init_responses=[{"code": 1, "output": error}]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.commands("terraform", "init")), 1) - self.assertEqual(self.sleeps(), []) - self.assertIn(error, self.output) - self.assert_no_imports() - - def test_state_read_permission_error_does_not_hide_unlock_failure(self): - error = f"{STATE_READ_PERMISSION_ERROR}\nError unlocking Azure state. Lock ID: test-lock" - result = self.run_bootstrap(init_responses=[{"code": 1, "output": error}]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.commands("terraform", "init")), 1) - self.assertEqual(self.sleeps(), []) - self.assertIn(error, self.output) - self.assert_no_imports() - def test_failed_unlock_with_403_is_visible_and_not_retried(self): error = ( "Error unlocking Azure state. Lock ID: test-lock\nError: failed to delete lock info from metadata: HTTP 403" @@ -974,35 +928,6 @@ def test_existing_lock_is_not_retried(self): self.assertEqual(self.sleeps(), []) self.assert_no_deployment() - def test_state_read_permission_failure_is_retried_before_plan(self): - result = self.run_deploy( - init_responses=[ - {"code": 1, "output": STATE_READ_PERMISSION_ERROR}, - {"code": 0, "output": "Backend ready"}, - ] - ) - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(len(self.commands("terraform", "init")), 2) - self.assertEqual(self.sleeps(), [10]) - self.assertIn(STATE_READ_PERMISSION_ERROR, self.output) - self.assert_deployed_once() - - def test_state_read_permission_exhaustion_stops_before_plan(self): - result = self.run_deploy(init_responses=[{"code": 1, "output": STATE_READ_PERMISSION_ERROR}]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.commands("terraform", "init")), 6) - self.assertEqual(self.sleeps(), [10, 20, 40, 80, 160]) - self.assert_no_deployment() - - def test_state_read_permission_error_does_not_hide_unlock_failure(self): - error = f"{STATE_READ_PERMISSION_ERROR}\nError unlocking Azure state. Lock ID: test-lock" - result = self.run_deploy(init_responses=[{"code": 1, "output": error}]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.commands("terraform", "init")), 1) - self.assertEqual(self.sleeps(), []) - self.assertIn(error, self.output) - self.assert_no_deployment() - def test_state_permission_error_is_not_retried(self): error = "Error loading state: HTTP 403" result = self.run_deploy(init_responses=[{"code": 1, "output": error}]) diff --git a/devops/tests/test_certificate_cache.py b/devops/tests/test_certificate_cache.py deleted file mode 100644 index ffcd3c8226..0000000000 --- a/devops/tests/test_certificate_cache.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Check certificate reuse using a local CA and mocked vault access.""" - -import base64 -from datetime import datetime, timedelta, timezone -import importlib.util -import os -from pathlib import Path -import shutil -import subprocess -import tempfile -import unittest -from unittest.mock import patch - -HELPER = Path(__file__).resolve().parents[2] / "templates/shared_services/certs/scripts/certificate_cache.py" -SPEC = importlib.util.spec_from_file_location("certificate_cache", HELPER) -CACHE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(CACHE) -FQDN = "nexus-tre.example.test" - - -class CertificateCacheTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.temporary = tempfile.TemporaryDirectory() - cls.addClassCleanup(cls.temporary.cleanup) - cls.root = Path(cls.temporary.name) - cls.command( - "req", - "-x509", - "-newkey", - "rsa:2048", - "-nodes", - "-days", - "365", - "-subj", - "/CN=Test Root", - "-addext", - "basicConstraints=critical,CA:TRUE", - "-keyout", - str(cls.root / "ca.key"), - "-out", - str(cls.root / "ca.pem"), - ) - for name, dns in (("valid", FQDN), ("wrong", "other.example.test"), ("wildcard", "*.example.test")): - folder = cls.root / name - folder.mkdir() - cls.command( - "req", - "-new", - "-newkey", - "rsa:2048", - "-nodes", - "-subj", - "/CN=" + dns, - "-keyout", - str(folder / "privkey.pem"), - "-out", - str(folder / "request.pem"), - ) - (folder / "extensions").write_text( - "basicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\n" - "extendedKeyUsage=serverAuth\nsubjectAltName=DNS:" + dns + "\n" - ) - cls.command( - "x509", - "-req", - "-in", - str(folder / "request.pem"), - "-CA", - str(cls.root / "ca.pem"), - "-CAkey", - str(cls.root / "ca.key"), - "-CAcreateserial", - "-days", - "90", - "-extfile", - str(folder / "extensions"), - "-out", - str(folder / "fullchain.pem"), - ) - (folder / "fullchain.pem").write_text( - (folder / "fullchain.pem").read_text() + (cls.root / "ca.pem").read_text() - ) - cls.command( - "pkcs12", - "-export", - "-inkey", - str(cls.root / "valid/privkey.pem"), - "-in", - str(cls.root / "valid/fullchain.pem"), - "-out", - str(cls.root / "valid.pfx"), - "-passout", - "pass:", - ) - - @staticmethod - def command(*args): - subprocess.run(["openssl", *args], capture_output=True, check=True, timeout=20) - - def setUp(self): - temporary = self.enterContext(tempfile.TemporaryDirectory()) - self.folder = Path(temporary) - for name in ("privkey.pem", "fullchain.pem"): - shutil.copyfile(self.root / "valid" / name, self.folder / name) - self.enterContext(patch.dict(os.environ, {"SSL_CERT_FILE": str(self.root / "ca.pem")})) - self.metadata = { - "attributes": {"enabled": True}, - "policy": {"secretProperties": {"contentType": "application/x-pkcs12"}}, - "sid": "https://vault.test/secrets/retained/exact-version", - "tags": {"tre-cache": CACHE.CACHE_TAG, "fqdn": FQDN}, - "x509ThumbprintHex": "ABC", - } - - def test_valid_certificate_with_matching_key_and_public_trust_is_reusable(self): - self.assertTrue(CACHE.validate_material(self.folder, FQDN)) - - def test_near_expiry_and_expired_material_need_renewal(self): - for days in (65, 95): - with self.subTest(days=days): - self.assertFalse( - CACHE.validate_material(self.folder, FQDN, datetime.now(timezone.utc) + timedelta(days=days)) - ) - - def test_not_yet_valid_material_is_rejected(self): - with self.assertRaisesRegex(ValueError, "not yet valid"): - CACHE.validate_material(self.folder, FQDN, datetime.now(timezone.utc) - timedelta(days=1)) - - def test_wrong_hostname_and_wildcard_are_rejected(self): - for name in ("wrong", "wildcard"): - with self.subTest(name=name), self.assertRaisesRegex(ValueError, "exact requested DNS"): - CACHE.validate_material(self.root / name, FQDN) - - def test_untrusted_chain_is_rejected(self): - with patch.dict(os.environ, {"SSL_CERT_FILE": str(self.folder / "missing-ca.pem")}, clear=False): - with self.assertRaises(RuntimeError): - CACHE.validate_material(self.folder, FQDN) - - def test_mismatched_or_missing_private_key_is_rejected(self): - shutil.copyfile(self.root / "wrong/privkey.pem", self.folder / "privkey.pem") - with self.assertRaisesRegex(ValueError, "do not match"): - CACHE.validate_material(self.folder, FQDN) - (self.folder / "privkey.pem").unlink() - with self.assertRaises(RuntimeError): - CACHE.validate_material(self.folder, FQDN) - - def test_cache_identity_depends_on_hostname_not_service_alias(self): - self.assertEqual(CACHE.cache_name(FQDN), CACHE.cache_name(FQDN.upper())) - self.assertNotEqual(CACHE.cache_name(FQDN), CACHE.cache_name("other.test")) - self.assertLessEqual(len(CACHE.cache_name(FQDN)), 127) - for name in ("nexus-ssl", "nexus-e2e-12345678"): - CACHE.validate_name(name) - with self.assertRaises(ValueError): - CACHE.validate_name(CACHE.cache_name(FQDN).upper()) - - def test_absent_cache_is_the_only_lookup_error_that_allows_issuance(self): - with patch.object(CACHE, "azure", side_effect=CACHE.CertificateNotFound()) as call: - self.assertFalse(CACHE.restore("vault", FQDN, self.folder)) - self.assertEqual(call.call_count, 2) - for error in ( - RuntimeError("Forbidden"), - RuntimeError("network failure"), - FileNotFoundError("Azure CLI is missing"), - ValueError("invalid JSON"), - ): - with self.subTest(error=error), patch.object(CACHE, "azure", side_effect=error): - with self.assertRaises(type(error)): - CACHE.restore("vault", FQDN, self.folder) - - def test_deleted_disabled_or_unowned_cache_is_not_overwritten(self): - with patch.object(CACHE, "azure", side_effect=[CACHE.CertificateNotFound(), {}]): - with self.assertRaisesRegex(RuntimeError, "deleted"): - CACHE.restore("vault", FQDN, self.folder) - for replacement in ( - {"attributes": {"enabled": False}}, - {"tags": {}}, - {"tags": {"tre-cache": CACHE.CACHE_TAG, "fqdn": "other.test"}}, - ): - metadata = self.metadata | replacement - with patch.object(CACHE, "azure", return_value=metadata), self.assertRaises(ValueError): - CACHE.restore("vault", FQDN, self.folder) - - def test_restore_downloads_exact_version_and_validates_real_pfx(self): - original = CACHE.run - calls = [] - - def command(args): - calls.append(args) - if args[:4] == ["az", "keyvault", "secret", "download"]: - shutil.copyfile(self.root / "valid.pfx", self.folder / "retained.pfx") - return "" - return original(args) - - with patch.object(CACHE, "azure", return_value=self.metadata), patch.object(CACHE, "run", side_effect=command): - self.assertTrue(CACHE.restore("vault", FQDN, self.folder)) - self.assertIn(self.metadata["sid"], calls[0]) - self.assertNotIn("--name", calls[0]) - - def test_store_to_restore_round_trip_preserves_pem_chain_and_key(self): - original = CACHE.run - retained = {} - metadata = self.metadata | {"policy": {"secretProperties": {"contentType": "application/x-pem-file"}}} - restored = self.folder / "restored" - restored.mkdir() - - def vault(*args): - if args[:3] == ("keyvault", "certificate", "import"): - retained["value"] = Path(args[args.index("--file") + 1]).read_text() - else: - self.assertEqual(args[:3], ("keyvault", "certificate", "show")) - return metadata - - def command(args): - if args[:4] == ["az", "keyvault", "secret", "download"]: - self.assertIn(metadata["sid"], args) - destination = Path(args[args.index("--file") + 1]) - encoding = args[args.index("--encoding") + 1] - if encoding == "base64": - destination.write_bytes(base64.b64decode(retained["value"])) - else: - self.assertEqual(encoding, "utf-8") - destination.write_text(retained["value"]) - return "" - return original(args) - - with patch.object(CACHE, "azure", side_effect=vault), patch.object(CACHE, "run", side_effect=command): - CACHE.store("vault", FQDN, self.folder) - self.assertTrue(CACHE.restore("vault", FQDN, restored)) - self.assertEqual((restored / "fullchain.pem").read_text(), (self.folder / "fullchain.pem").read_text()) - self.assertTrue(CACHE.validate_material(restored, FQDN)) - self.assertNotIn("PRIVATE KEY", (restored / "fullchain.pem").read_text()) - - def test_unknown_or_missing_content_type_stops_before_secret_download(self): - for policy in ({}, {"secretProperties": {"contentType": "text/plain"}}): - with ( - self.subTest(policy=policy), - patch.object(CACHE, "azure", return_value=self.metadata | {"policy": policy}), - ): - with patch.object(CACHE, "run") as command, self.assertRaisesRegex(ValueError, "content type"): - CACHE.restore("vault", FQDN, self.folder) - command.assert_not_called() - - def test_incomplete_or_mismatched_retained_pem_is_rejected(self): - original = CACHE.run - metadata = self.metadata | {"policy": {"secretProperties": {"contentType": "application/x-pem-file"}}} - chain = (self.folder / "fullchain.pem").read_text() - key = (self.folder / "privkey.pem").read_text() - wrong_key = (self.root / "wrong/privkey.pem").read_text() - for name, material in (("no-key", chain), ("no-chain", key), ("mismatched-key", wrong_key + chain)): - restored = self.folder / name - restored.mkdir() - - def command(args): - if args[:4] == ["az", "keyvault", "secret", "download"]: - Path(args[args.index("--file") + 1]).write_text(material) - return "" - return original(args) - - with self.subTest(name=name), patch.object(CACHE, "azure", return_value=metadata): - with patch.object(CACHE, "run", side_effect=command), self.assertRaises((ValueError, RuntimeError)): - CACHE.restore("vault", FQDN, restored) - - def test_store_checks_material_and_retains_before_alias_publication(self): - def import_certificate(*args): - self.assertEqual(args[:3], ("keyvault", "certificate", "import")) - self.assertIn(CACHE.cache_name(FQDN), args) - self.assertIn("fqdn=" + FQDN, args) - material = Path(args[args.index("--file") + 1]).read_text() - self.assertIn("PRIVATE KEY", material) - self.assertIn("CERTIFICATE", material) - return self.metadata - - with patch.object(CACHE, "azure", side_effect=import_certificate): - CACHE.store("vault", FQDN, self.folder) - self.assertFalse((self.folder / "cache-import.pem").exists()) - - def test_failed_cache_store_removes_temporary_import_and_does_not_claim_success(self): - with patch.object(CACHE, "azure", side_effect=RuntimeError("access denied")): - with self.assertRaises(RuntimeError): - CACHE.store("vault", FQDN, self.folder) - self.assertFalse((self.folder / "cache-import.pem").exists()) - - def test_command_errors_redact_secret_output_and_preserve_not_found_distinction(self): - for detail, expected in ( - ("ERROR: (CertificateNotFound) missing", CACHE.CertificateNotFound), - ("ERROR: (Forbidden) secret-password", RuntimeError), - ("secret-private-key", RuntimeError), - ): - with patch.object( - CACHE.subprocess, "run", side_effect=subprocess.CalledProcessError(1, ["az", "secret"], stderr=detail) - ): - with self.assertRaises(expected) as caught: - CACHE.run(["az", "secret"]) - self.assertNotIn("secret-password", str(caught.exception)) - self.assertNotIn("secret-private-key", str(caught.exception)) - with patch.object(CACHE.subprocess, "run", side_effect=subprocess.TimeoutExpired("private-command", 180)): - with self.assertRaisesRegex(RuntimeError, "timed out"): - CACHE.run(["az", "secret"]) - - def test_cli_guard_rejects_reserved_alias_before_terraform(self): - result = subprocess.run( - ["python3", str(HELPER), "validate-name", "--cert-name", "tre-acme-reserved"], - capture_output=True, - text=True, - timeout=10, - ) - self.assertEqual(result.returncode, 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/devops/tests/test_certificate_script.py b/devops/tests/test_certificate_script.py deleted file mode 100644 index 6cfe1329d1..0000000000 --- a/devops/tests/test_certificate_script.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Check issuance, cache failure and secret handling at the shell boundary.""" - -import json -import os -from pathlib import Path -import subprocess -import tempfile -import unittest - -ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / "templates/shared_services/certs/scripts/letsencrypt.sh" - - -class CertificateScriptTests(unittest.TestCase): - def execute(self, cache_status=0, issuance_status=0): - with tempfile.TemporaryDirectory() as temporary: - folder = Path(temporary) - log = folder / "calls.jsonl" - fake = folder / "fake" - fake.write_text("""#!/usr/bin/env python3 -import json,os,pathlib,sys -args=sys.argv[1:]; command=pathlib.Path(sys.argv[0]).name -with open(os.environ['CALL_LOG'],'a') as file: file.write(json.dumps([command,*args])+'\\n') -if command=='python3': - if 'restore' in args: sys.exit(int(os.environ['CACHE_STATUS'])) -elif command=='certbot': - if os.environ['ISSUANCE_STATUS']!='0': sys.exit(int(os.environ['ISSUANCE_STATUS'])) - d=pathlib.Path(args[args.index('--config-dir')+1])/'live'/args[args.index('--domain')+1];d.mkdir(parents=True) - for name in ('privkey.pem','fullchain.pem'): (d/name).write_text('test') -elif command=='openssl' and args[0]=='rand': print('private-password') -elif command=='az': - if args[:3]==['storage','blob','list']: print('[]') - elif args[:3]==['keyvault','certificate','import']: print(json.dumps({'sid':'https://vault.test/secrets/cert/version'})) - elif args[:3]==['keyvault','secret','set']: - assert '--output' in args and args[args.index('--output')+1]=='none' - assert pathlib.Path(args[args.index('--file')+1]).read_text()=='private-password' -""") - fake.chmod(0o700) - for name in ("python3", "certbot", "az", "openssl", "sleep"): - (folder / name).symlink_to(fake) - # The fake uses the host interpreter, not the intercepted python3 name. - fake.write_text(fake.read_text().replace("#!/usr/bin/env python3", "#!" + os.sys.executable)) - script = folder / "letsencrypt.sh" - script.write_text(SCRIPT.read_text().replace("/opt/certbot/bin/certbot", str(folder / "certbot"))) - result = subprocess.run( - [ - "bash", - str(script), - "--storage_account_name", - "storage", - "--fqdn", - "nexus.example.test", - "--keyvault_name", - "vault", - "--resource_group_name", - "rg", - "--application_gateway_name", - "agw", - "--cert_name", - "nexus-e2e-random", - "--password_name", - "password", - ], - env=os.environ - | { - "PATH": str(folder) + os.pathsep + os.environ["PATH"], - "CALL_LOG": str(log), - "CACHE_STATUS": str(cache_status), - "ISSUANCE_STATUS": str(issuance_status), - }, - capture_output=True, - text=True, - timeout=15, - ) - calls = [json.loads(line) for line in log.read_text().splitlines()] - restore = next(call for call in calls if "restore" in call) - work = Path(restore[restore.index("--directory") + 1]) - self.assertFalse(work.exists(), "Private temporary files must be removed on every exit") - self.assertNotIn("private-password", result.stdout + result.stderr) - return result, calls - - def test_reuse_does_not_call_acme_or_storage(self): - result, calls = self.execute() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse(any(call[0] == "certbot" or call[1:2] == ["storage"] for call in calls)) - self.assertTrue(any(call[1:4] == ["keyvault", "certificate", "import"] for call in calls)) - - def test_missing_cache_issues_once_and_persists_before_alias_update(self): - result, calls = self.execute(cache_status=3) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(sum(call[0] == "certbot" for call in calls), 1) - store = next(i for i, call in enumerate(calls) if "store" in call) - alias = next(i for i, call in enumerate(calls) if call[1:4] == ["keyvault", "certificate", "import"]) - self.assertLess(store, alias) - - def test_cache_failure_stops_before_acme_and_alias_changes(self): - result, calls = self.execute(cache_status=1) - self.assertEqual(result.returncode, 1) - self.assertEqual(len(calls), 1) - - def test_issuance_failure_does_not_update_cache_or_alias(self): - result, calls = self.execute(cache_status=3, issuance_status=1) - self.assertEqual(result.returncode, 1) - self.assertFalse(any("store" in call or call[1:3] == ["keyvault", "certificate"] for call in calls)) - - -if __name__ == "__main__": - unittest.main() diff --git a/devops/tests/test_certificate_storage_readiness.py b/devops/tests/test_certificate_storage_readiness.py deleted file mode 100644 index 06ff461b3b..0000000000 --- a/devops/tests/test_certificate_storage_readiness.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Check private certificate storage readiness without contacting Azure.""" - -import contextlib -import importlib.util -import io -import json -from pathlib import Path -import socket -import ssl -import subprocess -import sys -import unittest -from unittest.mock import MagicMock, patch - - -HELPER = Path(__file__).resolve().parents[2] / "templates/shared_services/certs/scripts/wait_for_private_storage.py" -SPEC = importlib.util.spec_from_file_location("certificate_storage_readiness", HELPER) -READINESS = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(READINESS) -HOSTNAME = "certificate.blob.core.windows.net" -PRIVATE_IP = "10.0.1.27" - - -class CertificateStorageReadinessTests(unittest.TestCase): - def test_private_address_uses_verified_tls_and_storage_hostname(self): - connection = MagicMock() - context = MagicMock() - answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PRIVATE_IP, 443))] - with ( - patch.object(READINESS.socket, "getaddrinfo", return_value=answers), - patch.object(READINESS.socket, "create_connection", return_value=connection) as connect, - patch.object(READINESS.ssl, "create_default_context", return_value=context), - ): - self.assertTrue(READINESS.probe(HOSTNAME, PRIVATE_IP)[0]) - connect.assert_called_once_with((PRIVATE_IP, 443), timeout=5) - context.wrap_socket.assert_called_once_with(connection.__enter__.return_value, server_hostname=HOSTNAME) - - def test_probe_requires_tls12_and_preserves_certificate_and_hostname_checks(self): - context = ssl.create_default_context() - context.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED - answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PRIVATE_IP, 443))] - with ( - patch.object(READINESS.socket, "getaddrinfo", return_value=answers), - patch.object(READINESS.socket, "create_connection", return_value=MagicMock()), - patch.object(READINESS.ssl, "create_default_context", return_value=context), - patch.object(context, "wrap_socket", return_value=MagicMock()), - ): - self.assertTrue(READINESS.probe(HOSTNAME, PRIVATE_IP)[0]) - self.assertEqual(context.minimum_version, ssl.TLSVersion.TLSv1_2) - self.assertEqual(context.verify_mode, ssl.CERT_REQUIRED) - self.assertTrue(context.check_hostname) - - def test_public_mixed_empty_and_wrong_private_answers_never_connect(self): - for addresses in ([], ["52.239.251.164"], [PRIVATE_IP, "52.239.251.164"], ["10.0.1.28"]): - with self.subTest(addresses=addresses): - answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (address, 443)) for address in addresses] - with ( - patch.object(READINESS.socket, "getaddrinfo", return_value=answers), - patch.object(READINESS.socket, "create_connection") as connect, - ): - self.assertFalse(READINESS.probe(HOSTNAME, PRIVATE_IP)[0]) - connect.assert_not_called() - - def test_lookup_failure_does_not_connect(self): - with ( - patch.object(READINESS.socket, "getaddrinfo", side_effect=socket.gaierror("diagnostic detail")), - patch.object(READINESS.socket, "create_connection") as connect, - ): - ready, reason = READINESS.probe(HOSTNAME, PRIVATE_IP) - self.assertFalse(ready) - self.assertIn("gaierror", reason) - self.assertNotIn("diagnostic detail", reason) - connect.assert_not_called() - - def test_tls_verification_failure_is_not_readiness(self): - answers = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PRIVATE_IP, 443))] - with ( - patch.object(READINESS.socket, "getaddrinfo", return_value=answers), - patch.object( - READINESS.socket, "create_connection", side_effect=ssl.SSLCertVerificationError("private diagnostic") - ), - ): - ready, reason = READINESS.probe(HOSTNAME, PRIVATE_IP) - self.assertFalse(ready) - self.assertIn("SSLCertVerificationError", reason) - self.assertNotIn("private diagnostic", reason) - - def wait_with_results(self, results, **kwargs): - elapsed = [0.0] - - def sleep(seconds): - elapsed[0] += seconds - - with ( - patch.object(READINESS.time, "monotonic", side_effect=lambda: elapsed[0]), - patch.object(READINESS.time, "sleep", side_effect=sleep), - patch.object(READINESS, "run_probe", side_effect=results) as probe, - contextlib.redirect_stdout(io.StringIO()), - ): - READINESS.wait_for_private_storage(HOSTNAME, PRIVATE_IP, **kwargs) - return probe.call_count, elapsed[0] - - def test_observed_dns_changes_restart_all_consecutive_checks(self): - # Private, lookup error, private, public, private, public, then stable. - results = [ - (ready, "private" if ready else "lookup or public DNS") - for ready in [True, False, True, False, True, False] + [True] * 7 - ] - count, elapsed = self.wait_with_results(results) - self.assertEqual(count, 13) - self.assertEqual(elapsed, 60) - - def test_sustained_readiness_spans_at_least_thirty_seconds(self): - count, elapsed = self.wait_with_results([(True, "ready")] * 7) - self.assertEqual(count, 7) - self.assertEqual(elapsed, 30) - - def test_tls_failure_also_restarts_stability(self): - results = [(True, "ready")] * 6 + [(False, "TLS check failed")] + [(True, "ready")] * 7 - count, elapsed = self.wait_with_results(results) - self.assertEqual(count, 14) - self.assertEqual(elapsed, 65) - - def test_timeout_prevents_unbounded_wait(self): - with self.assertRaisesRegex(TimeoutError, "within 12 seconds"): - self.wait_with_results([(False, "not ready")] * 3, timeout=12) - - def test_probe_gets_only_remaining_deadline(self): - elapsed = [0.0] - - def consume_budget(hostname, expected_ip, timeout): - self.assertEqual(timeout, 2) - elapsed[0] += timeout - return True, "too late" - - with ( - patch.object(READINESS.time, "monotonic", side_effect=lambda: elapsed[0]), - patch.object(READINESS, "run_probe", side_effect=consume_budget), - patch.object(READINESS.time, "sleep") as sleep, - ): - with self.assertRaises(TimeoutError): - READINESS.wait_for_private_storage(HOSTNAME, PRIVATE_IP, timeout=2, required_successes=1) - sleep.assert_not_called() - - def test_blocked_resolver_child_has_finite_timeout(self): - with patch.object(READINESS.subprocess, "run", side_effect=subprocess.TimeoutExpired("probe", 2)) as run: - ready, reason = READINESS.run_probe(HOSTNAME, PRIVATE_IP, 2) - self.assertFalse(ready) - self.assertIn("timed out", reason) - self.assertEqual(run.call_args.kwargs["timeout"], 2) - self.assertFalse(run.call_args.kwargs.get("shell", False)) - - def test_invalid_child_results_cannot_report_readiness(self): - for value, code in [ - ("not json", 0), - ("[]", 0), - ('{"ready":"yes","reason":"x"}', 0), - ('{"ready":true,"reason":"x"}', 1), - ("", 2), - ]: - with self.subTest(value=value, code=code): - result = subprocess.CompletedProcess([], code, value, "internal diagnostic") - with patch.object(READINESS.subprocess, "run", return_value=result): - ready, reason = READINESS.run_probe(HOSTNAME, PRIVATE_IP, 2) - self.assertFalse(ready) - self.assertNotIn("internal diagnostic", reason) - - def test_cli_rejects_public_and_special_expected_addresses(self): - for address in ("52.239.251.164", "127.0.0.1", "169.254.169.254", "0.0.0.0", "invalid"): - with self.subTest(address=address): - result = subprocess.run( - [sys.executable, str(HELPER), "--hostname", HOSTNAME, "--expected-ip", address], - capture_output=True, - text=True, - timeout=5, - check=False, - ) - self.assertEqual(result.returncode, 2) - self.assertIn("private endpoint IP address", result.stderr) - - def test_cli_failure_returns_nonzero(self): - with ( - patch.object(sys, "argv", [str(HELPER), "--hostname", HOSTNAME, "--expected-ip", PRIVATE_IP]), - patch.object(READINESS, "wait_for_private_storage", side_effect=TimeoutError("not ready")), - contextlib.redirect_stderr(io.StringIO()), - ): - self.assertEqual(READINESS.main(), 1) - - def test_probe_mode_reports_json_for_parent(self): - output = io.StringIO() - with ( - patch.object(sys, "argv", [str(HELPER), "--probe", "--hostname", HOSTNAME, "--expected-ip", PRIVATE_IP]), - patch.object(READINESS, "probe", return_value=(False, "not ready")), - contextlib.redirect_stdout(output), - ): - self.assertEqual(READINESS.main(), 1) - self.assertEqual(json.loads(output.getvalue()), {"ready": False, "reason": "not ready"}) - - -if __name__ == "__main__": - unittest.main() diff --git a/devops/tests/test_core_outputs.py b/devops/tests/test_core_outputs.py deleted file mode 100644 index c0c9bdbd5a..0000000000 --- a/devops/tests/test_core_outputs.py +++ /dev/null @@ -1,367 +0,0 @@ -"""Exercise core output failures through the real UI Make target, without Azure.""" - -import json -import os -from pathlib import Path -import re -import shlex -import shutil -import subprocess -import tempfile -import unittest - - -ROOT = Path(__file__).resolve().parents[2] -BASH = shutil.which("bash") -OUTPUT_NAMES = re.findall(r'^output "([^"]+)"', (ROOT / "core/terraform/outputs.tf").read_text(), re.MULTILINE) -VALID_OUTPUT = json.dumps( - { - name: {"value": "mockstorage" if name == "static_web_storage" else f"mock-{name}", "type": "string"} - for name in OUTPUT_NAMES - } -) -AUXILIARY_OUTPUTS = { - "WORKSPACE_API_CLIENT_ID": ("TEST_WORKSPACE_APP_ID",), - "WORKSPACE_API_CLIENT_SECRET": ("TEST_WORKSPACE_APP_SECRET",), - "SUB_ID": ("SUBSCRIPTION_ID", "AZURE_SUBSCRIPTION_ID"), - "TENANT_ID": ("AZURE_TENANT_ID",), -} - - -class CoreOutputTests(unittest.TestCase): - def setUp(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - self.root = Path(directory.name) - for relative in ( - "Makefile", - "config_schema.json", - "core/terraform/outputs.sh", - "core/terraform/json-to-env.sh", - "devops/scripts/load_env.sh", - "devops/scripts/load_and_validate_env.sh", - "devops/scripts/construct_tre_url.sh", - "devops/scripts/convert_azure_env_to_arm_env.sh", - ): - target = self.root / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(ROOT / relative, target) - self.core = self.root / "core" - self.log = self.root / "calls.log" - scripts = self.root / "devops/scripts" - scripts.mkdir(parents=True, exist_ok=True) - (scripts / "bootstrap_azure_env.sh").write_text("true\n") - (scripts / "storage_enable_public_access.sh").write_text( - 'echo storage-enable >> "$CALL_LOG"\ntrap \'echo storage-disable >> "$CALL_LOG"\' EXIT\n' - ) - deploy = scripts / "build_deploy_ui.sh" - deploy.write_text('#!/bin/bash\necho "ui:$STORAGE_ACCOUNT" >> "$CALL_LOG"\n') - deploy.chmod(0o755) - mock = self.root / "bin" - mock.mkdir() - terraform = mock / "terraform" - terraform.write_text( - '#!/bin/bash\necho "terraform:$1" >> "$CALL_LOG"\n' - 'if [ "$1" = init ]; then exit "${INIT_EXIT:-0}"; fi\n' - 'printf "%s" "$OUTPUT_JSON"\n' - 'if [ "${OUTPUT_EXIT:-0}" != 0 ]; then echo "Failed to load state" >&2; fi\n' - 'exit "${OUTPUT_EXIT:-0}"\n' - ) - terraform.chmod(0o755) - self.env = { - **os.environ, - "PATH": str(mock) + os.pathsep + os.environ["PATH"], - "CALL_LOG": str(self.log), - "OUTPUT_JSON": VALID_OUTPUT, - "INIT_EXIT": "0", - "OUTPUT_EXIT": "0", - "DEPLOY_UI": "true", - **dict.fromkeys(AUXILIARY_OUTPUTS, ""), - } - self.private = self.core / "private.env" - self.private.write_text("EXISTING='keep me'\n") - self.cache = self.core / "tre_output.json" - - def run_make(self): - result = subprocess.run( - [shutil.which("make"), "--no-print-directory", f"SHELL={BASH}", "build-and-deploy-ui"], - cwd=self.root, - env=self.env, - capture_output=True, - text=True, - timeout=10, - ) - self.calls = self.log.read_text().splitlines() - self.assertEqual(self.calls[-1], "storage-disable") - self.assertEqual(list(self.core.glob("*.tmp.*")), []) - return result - - def assert_stopped(self, result): - self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertFalse(any(call.startswith("ui:") for call in self.calls)) - self.assertEqual(self.private.read_text(), "EXISTING='keep me'\n") - - def test_failed_init_stops_before_output_and_ui(self): - self.env["INIT_EXIT"] = "17" - result = self.run_make() - self.assert_stopped(result) - self.assertNotIn("terraform:output", self.calls) - self.assertFalse(self.cache.exists()) - self.assertIn("Error 17", result.stderr) - - def test_failed_output_stops_ui_and_does_not_cache_partial_output(self): - for output in ("", '{"partial":', VALID_OUTPUT): - with self.subTest(output=output): - self.env.update(OUTPUT_EXIT="23", OUTPUT_JSON=output) - result = self.run_make() - self.assert_stopped(result) - self.assertFalse(self.cache.exists()) - self.assertIn("Failed to load state", result.stderr) - self.assertIn("Error 23", result.stderr) - - def test_invalid_successful_output_is_not_cached(self): - for output in ("", "{}", "[]", "null", '{"partial":'): - with self.subTest(output=output): - self.env["OUTPUT_JSON"] = output - result = self.run_make() - self.assert_stopped(result) - self.assertFalse(self.cache.exists()) - - def test_invalid_nonempty_cache_stops_ui(self): - self.cache.write_text('{"partial":') - result = self.run_make() - self.assert_stopped(result) - self.assertNotIn("terraform:output", self.calls) - - def test_incomplete_outputs_are_not_published(self): - incomplete_outputs = [{"unexpected": {"value": "x"}}, {"static_web_storage": {"value": "mockstorage"}}] - for missing in ("static_web_storage", "core_resource_group_name", "keyvault_name"): - output = json.loads(VALID_OUTPUT) - del output[missing] - incomplete_outputs.append(output) - for output in incomplete_outputs: - with self.subTest(output=output): - self.env["OUTPUT_JSON"] = json.dumps(output) - self.assert_stopped(self.run_make()) - self.assertFalse(self.cache.exists()) - - def test_invalid_mapped_values_are_not_published(self): - for value in ( - None, - "", - " \t\n", - "mock\nstorage", - "mock\rstorage", - "\x00mockstorage", - "mock\x00storage", - "mockstorage\x00", - "mock'storage", - 1, - False, - [], - {}, - ): - with self.subTest(value=value): - output = json.loads(VALID_OUTPUT) - output["static_web_storage"]["value"] = value - self.env["OUTPUT_JSON"] = json.dumps(output) - self.assert_stopped(self.run_make()) - self.assertFalse(self.cache.exists()) - - def test_missing_output_value_is_not_published(self): - output = json.loads(VALID_OUTPUT) - output["static_web_storage"] = {"type": "string"} - self.env["OUTPUT_JSON"] = json.dumps(output) - self.assert_stopped(self.run_make()) - self.assertFalse(self.cache.exists()) - - def test_invalid_auxiliary_values_preserve_existing_files(self): - marker = self.root / "injected" - invalid_values = ("mock'secret", "mock\nsecret", "mock\rsecret", f"secret'; touch {marker}; #") - for source_name in AUXILIARY_OUTPUTS: - for value in invalid_values: - for cached in (False, True): - with self.subTest(source_name=source_name, value=value, cached=cached): - self.env[source_name] = value - if cached: - self.cache.write_text(VALID_OUTPUT) - else: - self.cache.unlink(missing_ok=True) - result = self.run_make() - self.assert_stopped(result) - self.assertIn(source_name, result.stderr) - self.assertNotIn(value, result.stdout + result.stderr) - self.assertFalse(marker.exists()) - if cached: - self.assertEqual(self.cache.read_text(), VALID_OUTPUT) - else: - self.assertFalse(self.cache.exists()) - self.env[source_name] = "" - - def test_auxiliary_values_from_core_env_are_validated(self): - value = "mock'secret" - (self.core / ".env").write_text(f"WORKSPACE_API_CLIENT_SECRET={shlex.quote(value)}\n") - self.assert_stopped(self.run_make()) - self.assertFalse(self.cache.exists()) - - @unittest.skipUnless(shutil.which("pajv"), "The development container provides pajv for config validation") - def test_config_rejects_unsupported_secrets_before_shell_export(self): - config = { - "location": "westeurope", - "tre_id": "mocktre", - "authentication": { - "aad_tenant_id": "00000000-0000-0000-0000-000000000000", - "workspace_api_client_secret": "mock-safe-secret", - }, - } - config_file = self.root / "config.yaml" - config_file.write_text(json.dumps(config)) - valid = subprocess.run( - [shutil.which("pajv"), "validate", "-s", "config_schema.json", "-d", "config.yaml"], - cwd=self.root, - capture_output=True, - text=True, - timeout=10, - ) - self.assertEqual(valid.returncode, 0, valid.stdout + valid.stderr) - yq = self.root / "bin/yq" - yq.write_text('#!/bin/bash\necho yq >> "$CALL_LOG"\nexit 97\n') - yq.chmod(0o755) - for character in ("'", "\r", "\n", "\x00"): - for value in (f"mock{character}secret-value", f"mock-secret-value{character}"): - with self.subTest(value=value): - config["authentication"]["workspace_api_client_secret"] = value - config_file.write_text(json.dumps(config)) - result = subprocess.run( - [BASH, "-c", '. "$DIR/load_and_validate_env.sh"'], - cwd=self.root, - env={**self.env, "DIR": str(self.root / "devops/scripts")}, - capture_output=True, - text=True, - timeout=10, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("config.yaml is invalid", result.stdout) - self.assertNotIn(value, result.stdout + result.stderr) - self.assertFalse(self.log.exists(), "Config validation must fail before yq exports values") - self.assertEqual(self.private.read_text(), "EXISTING='keep me'\n") - - def test_auxiliary_values_match_both_environment_loaders(self): - output_names = [name for names in AUXILIARY_OUTPUTS.values() for name in names] - print_values = 'printf "%s\\0" ' + " ".join(f'"${{{name}}}"' for name in output_names) - for value in ("", ' \tmock"\\$secret; `printf injected`; $(printf injected) == \t '): - with self.subTest(value=value): - self.env.update(dict.fromkeys(AUXILIARY_OUTPUTS, value)) - result = self.run_make() - self.assertEqual(result.returncode, 0, result.stderr) - for loader in ("load_env.sh", "load_and_validate_env.sh"): - with self.subTest(loader=loader): - result = subprocess.run( - [ - BASH, - "-c", - '. "$1" "$2"; ' + print_values, - "bash", - str(self.root / "devops/scripts" / loader), - str(self.private), - ], - cwd=self.root, - env={ - **self.env, - "DIR": str(self.root / "devops/scripts"), - "USE_ENV_VARS_NOT_FILES": "true", - }, - capture_output=True, - text=True, - timeout=10, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.split("\x00"), [value] * len(output_names) + [""]) - - def test_incomplete_cache_preserves_existing_files(self): - invalid_cache = '{"unexpected":{"value":"x"}}' - self.cache.write_text(invalid_cache) - self.assert_stopped(self.run_make()) - self.assertEqual(self.cache.read_text(), invalid_cache) - self.assertNotIn("terraform:output", self.calls) - - def test_multiple_output_documents_are_not_published(self): - self.env["OUTPUT_JSON"] = VALID_OUTPUT + "\n" + VALID_OUTPUT - self.assert_stopped(self.run_make()) - self.assertFalse(self.cache.exists()) - - def test_successful_output_reaches_ui_with_storage_account(self): - result = self.run_make() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("ui:mockstorage", self.calls) - self.assertEqual(json.loads(self.cache.read_text()), json.loads(VALID_OUTPUT)) - self.assertIn("STORAGE_ACCOUNT='mockstorage'", self.private.read_text()) - - def test_valid_cache_avoids_terraform(self): - self.cache.write_text(VALID_OUTPUT) - result = self.run_make() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertFalse(any(call.startswith("terraform:") for call in self.calls)) - self.assertIn("ui:mockstorage", self.calls) - - def test_legacy_optional_output_is_preserved_when_present(self): - output = json.loads(VALID_OUTPUT) - output["airlock_malware_scan_result_topic_name"] = {"value": "legacy-topic"} - self.cache.write_text(json.dumps(output)) - result = self.run_make() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("AIRLOCK_MALWARE_SCAN_RESULT_TOPIC_NAME='legacy-topic'", self.private.read_text()) - - def test_output_values_match_environment_loaders(self): - output = json.loads(VALID_OUTPUT) - value = 'mock"\\$storage; echo injected' - output["static_web_storage"]["value"] = value - self.env["OUTPUT_JSON"] = json.dumps(output) - result = self.run_make() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn(f"ui:{value}", self.calls) - self.assertNotIn("injected", result.stdout) - # load_and_validate_env.sh sources private.env directly. - sourced = subprocess.run( - [BASH, "-c", '. "$1"; printf "%s" "$STORAGE_ACCOUNT"', "bash", str(self.private)], - capture_output=True, - text=True, - timeout=10, - ) - self.assertEqual(sourced.returncode, 0, sourced.stderr) - self.assertEqual(sourced.stdout, value) - - def test_empty_cache_is_regenerated(self): - self.cache.write_text("") - result = self.run_make() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("terraform:output", self.calls) - self.assertIn("ui:mockstorage", self.calls) - - def test_conversion_failure_preserves_environment_and_retries(self): - output = json.loads(VALID_OUTPUT) - output["static_web_storage"] = 1 - self.env["OUTPUT_JSON"] = json.dumps(output) - result = self.run_make() - self.assert_stopped(result) - self.assertFalse(self.cache.exists()) - terraform_outputs_before_retry = self.calls.count("terraform:output") - - self.env["OUTPUT_JSON"] = VALID_OUTPUT - retry = self.run_make() - self.assertEqual(retry.returncode, 0, retry.stderr) - self.assertEqual(self.calls.count("terraform:output"), terraform_outputs_before_retry + 1) - self.assertIn("ui:mockstorage", self.calls) - - def test_converter_reports_invalid_json(self): - for output in ("", "{}", "[]", "null", '{"partial":', VALID_OUTPUT + "\n" + VALID_OUTPUT): - with self.subTest(output=output): - result = subprocess.run( - [BASH, "core/terraform/json-to-env.sh"], - cwd=self.root, - input=output, - capture_output=True, - text=True, - timeout=10, - ) - self.assertNotEqual(result.returncode, 0) diff --git a/devops/tests/test_mgmt_storage_private_endpoint.py b/devops/tests/test_mgmt_storage_private_endpoint.py deleted file mode 100644 index 5b5d70c154..0000000000 --- a/devops/tests/test_mgmt_storage_private_endpoint.py +++ /dev/null @@ -1,262 +0,0 @@ -"""Exercise the core management storage private endpoint step with mocked tools.""" - -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest - - -SCRIPT = Path(__file__).resolve().parents[2] / "core/terraform/mgmt_storage_private_endpoint.sh" -PE_ADDRESS = "module.resource_processor_vmss_porter[0].azurerm_private_endpoint.mgmtblobpe" -HOST = "mockstorage.blob.core.windows.net" - -MOCK_COMMAND = r""" -import json -import os -from pathlib import Path -import sys - -root = Path(os.environ["MOCK_ROOT"]) -config = json.loads((root / "config.json").read_text()) -state_file = root / "state.json" -state = json.loads(state_file.read_text()) if state_file.exists() else {"lookups": 0} -args = sys.argv[1:] -command = Path(sys.argv[0]).name -with (root / "calls.jsonl").open("a") as log: - log.write(json.dumps([command, *args]) + "\n") - - -def finish(code=0, output=""): - state_file.write_text(json.dumps(state)) - if output: - print(output, file=sys.stderr if code else sys.stdout) - sys.exit(code) - - -if command == "sleep": - finish() -if command == "timeout": - os.execvp(args[1], args[1:]) -if command == "az" and args[:3] == ["storage", "account", "show"]: - finish(output="https://" + config["host"] + "/") -if command == "az" and args[:3] == ["storage", "blob", "exists"]: - ready = config["backend"][min(state["lookups"], len(config["backend"])) - 1] - finish(0 if ready else 1, config["exists"] if ready else config["backend_error"]) -if command == "terraform" and args[0] == "plan": - finish(config["plan_exit"]) -if command == "terraform" and args[0] == "apply": - finish(config["apply_exit"], "apply failed" if config["apply_exit"] else "") -if command == "dig": - if "CNAME" not in args: - finish(output="DNS diagnostics") - state["lookups"] += 1 - alias = config["cnames"][min(state["lookups"], len(config["cnames"])) - 1] - finish(output=alias) -if command == "getent": - if state["lookups"] == 0: - finish(output="20.0.0.1 STREAM " + args[1]) - answer = config["dns"][min(state["lookups"], len(config["dns"])) - 1] - if answer: - finish(output="10.1.2.3 STREAM " + args[1]) - finish(2) -finish(99, "Unexpected mocked command: " + repr([command, *args])) -""" - - -class MgmtStoragePrivateEndpointTests(unittest.TestCase): - def run_script( - self, - plan_exit=2, - dns=(True,), - apply_exit=0, - rp_type="vmss_porter", - cnames=None, - backend=(True,), - host=HOST, - exists="true", - backend_error="backend unavailable", - ): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - (root / "config.json").write_text( - json.dumps( - { - "plan_exit": plan_exit, - "dns": list(dns), - "apply_exit": apply_exit, - "cnames": cnames or [host.replace(".", ".privatelink.", 1) + "."], - "backend": backend, - "host": host, - "exists": exists, - "backend_error": backend_error, - } - ) - ) - for name in ("az", "terraform", "getent", "sleep", "dig", "timeout"): - executable = root / name - executable.write_text(f"#!{sys.executable}\n" + MOCK_COMMAND) - executable.chmod(0o755) - env = { - **os.environ, - "MOCK_ROOT": str(root), - "PATH": str(root) + os.pathsep + os.environ["PATH"], - "TF_VAR_mgmt_storage_account_name": "mockstorage", - "TF_VAR_mgmt_resource_group_name": "mockgroup", - "TF_VAR_terraform_state_container_name": "tfstate", - "TRE_ID": "mock-tre", - "TF_VAR_resource_processor_type": rp_type, - } - result = subprocess.run( - ["/bin/bash", str(SCRIPT)], - cwd=root, - env=env, - text=True, - capture_output=True, - timeout=60, - ) - calls_file = root / "calls.jsonl" - self.calls = ( - [json.loads(line) for line in calls_file.read_text().splitlines()] if calls_file.exists() else [] - ) - self.output = result.stdout + result.stderr - return result - - def terraform_calls(self, subcommand): - return [c for c in self.calls if c[0] == "terraform" and c[1] == subcommand] - - def dns_attempts(self): - return [c for c in self.calls if c[0] == "dig" and "CNAME" in c] - - def blob_read_calls(self): - return [c for c in self.calls if c[:4] == ["az", "storage", "blob", "exists"]] - - def test_unchanged_private_endpoint_skips_apply_but_checks_readiness(self): - result = self.run_script(plan_exit=0) - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(len(self.terraform_calls("plan")), 1) - self.assertEqual(self.terraform_calls("apply"), []) - self.assertEqual(len(self.dns_attempts()), 3) - self.assertEqual(len(self.blob_read_calls()), 3) - self.assertEqual(self.terraform_calls("state"), []) - for call in self.blob_read_calls(): - self.assertEqual(call[call.index("--auth-mode") + 1], "login") - self.assertEqual(call[call.index("--container-name") + 1], "tfstate") - self.assertEqual(call[call.index("--name") + 1], "mock-tre") - - def test_missing_state_blob_is_reachable_without_initialising_state(self): - result = self.run_script(plan_exit=0, exists="false") - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(self.terraform_calls("state"), []) - self.assertEqual(len(self.blob_read_calls()), 3) - self.assertEqual([c for c in self.calls if c[:3] == ["az", "storage", "blob"]], self.blob_read_calls()) - - def test_denied_blob_access_cannot_count_as_a_missing_blob(self): - result = self.run_script(backend=(False,), backend_error="AuthorizationPermissionMismatch: HTTP 403") - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.blob_read_calls()), 30) - self.assertIn("AuthorizationPermissionMismatch", self.output) - - def test_missing_or_unexpected_blob_result_cannot_pass_readiness(self): - for exists in ("", "null", "unexpected"): - with self.subTest(exists=exists): - result = self.run_script(exists=exists) - self.assertNotEqual(result.returncode, 0) - self.assertIn("Unexpected response", self.output) - - def test_rerun_still_fails_if_dns_never_becomes_ready(self): - result = self.run_script(plan_exit=0, dns=(False,)) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(self.terraform_calls("apply"), []) - self.assertEqual(len(self.dns_attempts()), 30) - - def test_stale_public_alias_cannot_satisfy_readiness(self): - result = self.run_script(cnames=["blob.storage.example.test."]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.dns_attempts()), 30) - self.assertEqual(self.blob_read_calls(), []) - self.assertIn("expected alias not observed", self.output) - - def test_stale_alias_resets_consecutive_successes(self): - private = "mockstorage.privatelink.blob.core.windows.net." - result = self.run_script(cnames=[private, "stale.example.test.", private, private, private]) - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(len(self.dns_attempts()), 5) - - def test_another_accounts_private_alias_is_rejected(self): - result = self.run_script(cnames=["other.privatelink.blob.core.windows.net."]) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.dns_attempts()), 30) - - def test_alias_uses_actual_cloud_endpoint_and_dns_case_insensitivity(self): - result = self.run_script( - host="mockstorage.blob.core.usgovcloudapi.net", - cnames=["MOCKSTORAGE.PRIVATELINK.BLOB.CORE.USGOVCLOUDAPI.NET."], - ) - self.assertEqual(result.returncode, 0, self.output) - - def test_backend_failure_resets_and_recovers_without_logging_state(self): - result = self.run_script(backend=(True, False, True, True, True)) - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(len(self.dns_attempts()), 5) - self.assertIn("backend read failed", self.output) - self.assertNotIn("SECRET_STATE_CONTENT", self.output) - - def test_persistent_backend_failure_is_bounded_and_has_dns_diagnostics(self): - result = self.run_script(backend=(False,)) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.dns_attempts()), 30) - self.assertIn("backend unavailable", self.output) - self.assertTrue(any(c[0] == "dig" and "@168.63.129.16" in c for c in self.calls)) - - def test_other_resource_processor_type_skips(self): - result = self.run_script(rp_type="other") - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(self.calls, []) - - def test_changed_target_is_applied_before_dns_checks(self): - result = self.run_script(dns=(True, True, True)) - self.assertEqual(result.returncode, 0, self.output) - plans = self.terraform_calls("plan") - self.assertEqual(len(plans), 1) - self.assertIn(f"-target={PE_ADDRESS}", plans[0]) - self.assertIn("-detailed-exitcode", plans[0]) - applies = self.terraform_calls("apply") - self.assertEqual(len(applies), 1) - self.assertEqual(applies[0][-1], plans[0][plans[0].index("-out") + 1]) - self.assertLess(self.calls.index(applies[0]), self.calls.index(self.dns_attempts()[0])) - self.assertEqual(len(self.dns_attempts()), 3) - self.assertEqual([c for c in self.calls if c[0] == "sleep"], [["sleep", "10"]] * 2) - self.assertIn(HOST, self.output) - - def test_transient_dns_failure_resets_and_recovers(self): - result = self.run_script(dns=(True, False, False, True, True, True)) - self.assertEqual(result.returncode, 0, self.output) - self.assertEqual(len(self.dns_attempts()), 6) - self.assertIn("did not resolve", self.output) - - def test_persistent_dns_failure_is_bounded_and_reported(self): - result = self.run_script(dns=(False,)) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(len(self.dns_attempts()), 30) - self.assertEqual(len([c for c in self.calls if c[0] == "sleep"]), 29) - self.assertIn(f"{HOST} did not become ready after 30 attempts", self.output) - - def test_targeted_apply_failure_stops_before_dns_wait(self): - result = self.run_script(apply_exit=1) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(self.dns_attempts(), []) - self.assertIn("apply failed", self.output) - - def test_targeted_plan_failure_stops_without_applying_or_waiting(self): - result = self.run_script(plan_exit=1) - self.assertEqual(result.returncode, 1) - self.assertEqual(self.terraform_calls("apply"), []) - self.assertEqual(self.dns_attempts(), []) - self.assertIn("plan failed (exit 1)", self.output) - - -if __name__ == "__main__": - unittest.main() diff --git a/devops/tests/test_review_vm_prerequisites.py b/devops/tests/test_review_vm_prerequisites.py deleted file mode 100644 index 5af9424b95..0000000000 --- a/devops/tests/test_review_vm_prerequisites.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Exercise the production review-VM bootstrap with mocked Windows/network commands.""" - -import json -import os -from pathlib import Path -import shutil -import subprocess -import tempfile -import unittest - - -ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ( - ROOT / "templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1" -) -HARNESS = r""" -$ErrorActionPreference = "Stop" -$settings = $env:BOOTSTRAP_TEST_SETTINGS | ConvertFrom-Json -$source = Get-Content -Raw $env:BOOTSTRAP_SOURCE -$source = $source.Replace('${nexus_proxy_url}', 'https://nexus.example.test') -$parseErrors = $null -$tokens = $null -$ast = [System.Management.Automation.Language.Parser]::ParseInput($source, [ref]$tokens, [ref]$parseErrors) -if ($parseErrors) { throw $parseErrors[0] } -$definition = $ast.Find({ param($node) - $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq 'Install-TreTool' -}, $true) -Invoke-Expression $definition.Extent.Text -function Wait-ForNexus { return $settings.nexus_ready } -function Configure-RProxy {} -function Configure-CondaProxy {} -function Start-Sleep {} -function Invoke-WebRequest { - param($Uri, $Method, $OutFile, [switch]$UseBasicParsing, $TimeoutSec) - if ($Method -eq 'Head' -and $settings.package_missing) { throw 'PACKAGE_MISSING' } - if ($OutFile -and $settings.download_failed) { throw 'DOWNLOAD_FAILED' } -} -function Start-Process { - param($FilePath, $ArgumentList, [switch]$Wait, [switch]$PassThru, [switch]$NoNewWindow) - return [pscustomobject]@{ ExitCode = $settings.exit_code } -} -function Get-Command { - param($Name, $ErrorAction) - if (-not $settings.cli_missing) { return [pscustomobject]@{Name = 'az'} } -} -function az { $global:LASTEXITCODE = $settings.cli_exit_code } -$NexusActionsEnabled = $true -$RequireAzureCli = $settings.required -$InstallAzureCli = $true -$ConfigureConda = $false -$AzureCliVersion = '2.88.0' -$ToolsDir = 'test-tools' -$start = $source.IndexOf('# Make sure the proxy is up') -$end = $source.IndexOf('$VsCodeSetup =') -Invoke-Expression $source.Substring($start, $end - $start) -Write-Host 'BOOTSTRAP_OK' -""" - - -@unittest.skipUnless(shutil.which("pwsh"), "PowerShell is required to execute the Windows bootstrap regression tests") -class ReviewVmPrerequisiteTests(unittest.TestCase): - def run_bootstrap(self, **overrides): - settings = { - "required": True, - "nexus_ready": True, - "package_missing": False, - "download_failed": False, - "exit_code": 0, - "cli_missing": False, - "cli_exit_code": 0, - **overrides, - } - with tempfile.TemporaryDirectory() as folder: - harness = Path(folder) / "bootstrap-test.ps1" - harness.write_text(HARNESS) - result = subprocess.run( - ["pwsh", "-NoLogo", "-NoProfile", "-File", str(harness)], - env={**os.environ, "BOOTSTRAP_SOURCE": str(SCRIPT), "BOOTSTRAP_TEST_SETTINGS": json.dumps(settings)}, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - return result - - def test_missing_nexus_or_package_fails_before_install(self): - for changes, message in ( - ({"nexus_ready": False}, "reachable Nexus"), - ({"package_missing": True}, "package is unavailable"), - ): - with self.subTest(changes=changes): - result = self.run_bootstrap(**changes) - self.assertNotEqual(result.returncode, 0) - self.assertIn(message, result.stderr) - self.assertNotIn("Installing Azure CLI", result.stdout) - - def test_required_cli_download_installer_and_execution_failures_stop_bootstrap(self): - for changes, message in ( - ({"download_failed": True}, "could not be installed"), - ({"exit_code": 1603}, "could not be installed"), - ({"cli_missing": True}, "executable was not found"), - ({"cli_exit_code": 1}, "failed its version check"), - ): - with self.subTest(changes=changes): - result = self.run_bootstrap(**changes) - self.assertNotEqual(result.returncode, 0) - self.assertIn(message, result.stderr) - self.assertNotIn("BOOTSTRAP_OK", result.stdout) - - def test_success_and_msi_reboot_required_code_are_accepted(self): - for code in (0, 3010): - with self.subTest(code=code): - result = self.run_bootstrap(exit_code=code) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("BOOTSTRAP_OK", result.stdout) - - def test_optional_cli_preserves_best_effort_behaviour(self): - for changes in ({"nexus_ready": False, "download_failed": True}, {"exit_code": 1603}): - with self.subTest(changes=changes): - result = self.run_bootstrap(required=False, **changes) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("WARNING:", result.stdout) - self.assertIn("BOOTSTRAP_OK", result.stdout) diff --git a/devops/version.txt b/devops/version.txt index c69cd12ff3..69cc2a5374 100644 --- a/devops/version.txt +++ b/devops/version.txt @@ -1 +1 @@ -__version__ = "0.6.15" +__version__ = "0.6.18" diff --git a/docs/tre-admins/setup-instructions/configuring-shared-services.md b/docs/tre-admins/setup-instructions/configuring-shared-services.md index b81880de35..2fb39f441b 100644 --- a/docs/tre-admins/setup-instructions/configuring-shared-services.md +++ b/docs/tre-admins/setup-instructions/configuring-shared-services.md @@ -30,30 +30,6 @@ Once deployed, the certs service will use Letsencrypt to generate a certificate You can verify whether this has been successful by navigating to your core keyvault (`kv-{TRE_ID}`) and looking for a certificate called `nexus-ssl` (or whatever you called it). -### Certificate reuse and renewal - -From certificate bundle `0.7.16`, install and `renew` reuse a retained certificate when more than 30 days remain before expiry. -Version `0.7.16` can also reuse certificates retained by version `0.7.15`. -The bundle verifies public trust, the exact DNS name and the private key before reuse. -It issues a new certificate when no retained certificate exists or renewal is due. -A vault lookup, ownership or certificate validation failure stops the operation. It does not trigger another issuance attempt. - -The core Key Vault retains one certificate named `tre-acme-` for each hostname. -The `tre-cache=certificate-v1` and `fqdn` tags identify this certificate. -The certificate includes an exportable private key and uses the existing vault access controls. -Names starting with `tre-acme-` are reserved. Use another name for the service certificate. - -Uninstall removes the service certificate and its infrastructure. It retains the hostname certificate for later installs, including installs with a different service certificate name. -Deleting the core vault removes this retained material under the vault's existing soft-delete and purge-protection rules. -If an administrator disables or deletes the retained certificate, investigate that change before enabling or recovering it. -Do not remove the cache to bypass an issuance limit. Compromised certificates require revocation and replacement through the normal incident procedure. - -The cache starts with the first successful issuance by a bundle that supports certificate retention. It does not search for or adopt uncached certificates. -An ACME rate-limit response still fails the operation. Follow the retry time in that response. -Run only one certificate lifecycle or dependent test suite per environment at a time. - -### Deploy Nexus - After verifying the certificate has been generated, you can deploy Nexus: 1. Run the below command in your terminal to build, publish and register the Nexus shared service bundle: diff --git a/resource_processor/_version.py b/resource_processor/_version.py index 23a4e3cb02..17b059f281 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.13.14" +__version__ = "0.13.12" diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py index 857204caa1..974044b782 100644 --- a/resource_processor/tests_rp/test_runner.py +++ b/resource_processor/tests_rp/test_runner.py @@ -492,55 +492,24 @@ async def test_invoke_porter_action_upgrade_failure_install_failure( @pytest.mark.asyncio -@pytest.mark.parametrize( - "action, expected_status", - [ - ("install", "deployment_failed"), - ("upgrade", "updating_failed"), - ("uninstall", "deleting_failed"), - ("custom-action", "action_failed"), - ], -) -@pytest.mark.parametrize( - "error", - [ - ( - "could not find installation /test_id: could not read storage schema document: " - + "rpc error: code = DeadlineExceeded desc = server selection error: context deadline exceeded" - ), - "could not find installation /test_id: installation does not exist", - "", - None, - ], - ids=["storage-timeout", "missing-installation", "empty-error", "no-error"], -) -async def test_invoke_porter_action_nonzero_exit_without_confirmed_absence_reports_failure( - action, expected_status, error +@patch("vmss_porter.runner.build_porter_command", return_value=([["porter", "install"]], None, "tre-params-test", None)) +@patch("vmss_porter.runner.run_porter", return_value=(1, "", "could not find installation")) +@patch("vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action_uninstall_failure( + mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client ): - sender = AsyncMock() - client = Mock() - client.get_queue_sender.return_value = sender + """Test invoking a porter action with uninstall failure.""" + mock_sb_client = AsyncMock(spec=ServiceBusClient) + mock_sb_sender = AsyncMock() + mock_sb_client.get_queue_sender.return_value = mock_sb_sender + config = {"deployment_status_queue": "test_queue"} - msg_body = {"id": "test_id", "action": action, "stepId": "test_step_id", "operationId": "test_operation_id"} + msg_body = {"id": "test_id", "action": "uninstall", "stepId": "test_step_id", "operationId": "test_operation_id"} - with ( - patch("vmss_porter.runner.build_porter_command", return_value=([["porter", action]], None, "params", None)), - patch("vmss_porter.runner.run_porter", return_value=(1, "", error)), - patch("vmss_porter.runner.run_command_helper", return_value=(1, None, "Storage is unavailable")) as lookup, - patch("vmss_porter.runner.get_porter_outputs", return_value=(True, {})) as outputs, - ): - result = await invoke_porter_action(msg_body, client, config) + result = await invoke_porter_action(msg_body, mock_sb_client, config) - assert result is False - outputs.assert_not_awaited() - assert sender.send_messages.await_count == 2 - final_status = json.loads(str(sender.send_messages.call_args.args[0])) - assert final_status["status"] == expected_status - assert (error or "Porter exited with code 1 and no error output.") in final_status["message"] - if action == "uninstall" and "could not find installation" in (error or ""): - lookup.assert_awaited_once() - else: - lookup.assert_not_awaited() + assert result is True + mock_sb_sender.send_messages.assert_called() @pytest.mark.asyncio @@ -819,151 +788,6 @@ async def test_invoke_porter_action_reports_legacy_guard_and_cleans_up(tmp_path) assert "Upgrade this resource" in final_status["message"] -@pytest.mark.asyncio -async def test_uninstall_storage_failure_after_parameter_check_preserves_failure(tmp_path): - parameters = tmp_path / "parameters.json" - installation = tmp_path / "installation.json" - parameters.write_text("parameter references") - installation.write_text("installation document") - action = ["porter", "uninstall", "resource"] - config = { - "azure_environment": "AzureCloud", - "vmss_msi_id": "msi", - "registry_server": "registry.azurecr.io", - "deployment_status_queue": "status", - } - msg_body = {"id": "resource", "action": "uninstall", "operationId": "operation", "stepId": "step"} - sender = AsyncMock() - client = Mock() - client.get_queue_sender.return_value = sender - error = "could not find installation /resource: could not read storage schema document: DeadlineExceeded" - responses = [(0, None, None)] * 5 + [ - (0, '[{"name":"resource"}]', None), - (0, '{"name":"resource","status":{"runId":"old-run"},"parameters":{}}', None), - (1, None, error), - (0, None, None), - (1, None, "Storage is still unavailable"), - ] - with ( - patch( - "vmss_porter.runner.build_porter_command", - return_value=([action], str(parameters), "test-parameters", str(installation)), - ), - patch("vmss_porter.runner.run_command_helper", new_callable=AsyncMock, side_effect=responses) as command, - patch("vmss_porter.runner.get_porter_outputs") as outputs, - ): - result = await invoke_porter_action(msg_body, client, config) - - assert result is False - assert not parameters.exists() - assert not installation.exists() - assert command.await_args_list[-3].args[0] == action - assert command.await_args_list[-2].args[0] == ["porter", "parameters", "delete", "test-parameters"] - assert command.await_args_list[-1].args[0] == [ - "porter", - "installations", - "list", - "--name", - "resource", - "--output", - "json", - "--verbosity", - "error", - ] - outputs.assert_not_awaited() - final_status = json.loads(str(sender.send_messages.call_args.args[0])) - assert final_status["status"] == "deleting_failed" - assert error in final_status["message"] - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "confirmation, expected_success", - [ - pytest.param((0, "[]", None), True, id="confirmed-absent"), - pytest.param((0, '[{"name":"resource"}]', None), False, id="installation-exists"), - pytest.param((0, '[{"name":"another-resource"}]', None), False, id="unexpected-installation"), - pytest.param((1, "[]", "Storage is unavailable"), False, id="failed-empty-lookup"), - pytest.param((0, "invalid JSON", None), False, id="invalid-json"), - pytest.param((0, "{}", None), False, id="object-response"), - pytest.param((0, "null", None), False, id="null-response"), - pytest.param((0, "", None), False, id="empty-output"), - pytest.param((0, None, None), False, id="no-output"), - pytest.param(RuntimeError("Lookup failed"), False, id="lookup-exception"), - pytest.param(TimeoutError(), False, id="lookup-timeout"), - pytest.param(asyncio.CancelledError(), None, id="cancelled-lookup"), - ], -) -async def test_uninstall_missing_installation_requires_fresh_confirmation(tmp_path, confirmation, expected_success): - parameters = tmp_path / "parameters.json" - installation = tmp_path / "installation.json" - parameters.write_text("parameter references") - installation.write_text("installation document") - action = ["porter", "uninstall", "resource"] - config = { - "azure_environment": "AzureCloud", - "vmss_msi_id": "msi", - "registry_server": "registry.azurecr.io", - "deployment_status_queue": "status", - } - message = {"id": "resource", "action": "uninstall", "operationId": "operation", "stepId": "step"} - sender = AsyncMock() - client = Mock() - client.get_queue_sender.return_value = sender - error = "could not find installation /resource: installation does not exist" - # The parameter check already saw an empty list. It cannot replace the fresh - # lookup after uninstall fails, because storage availability can change. - responses = [(0, None, None)] * 5 + [ - (0, "[]", None), - (1, None, error), - (0, None, None), - confirmation, - ] - with ( - patch( - "vmss_porter.runner.build_porter_command", - return_value=([action], str(parameters), "test-parameters", str(installation)), - ), - patch("vmss_porter.runner.run_command_helper", new_callable=AsyncMock, side_effect=responses) as command, - patch("vmss_porter.runner.get_porter_outputs") as outputs, - ): - if expected_success is None: - with pytest.raises(asyncio.CancelledError): - await invoke_porter_action(message, client, config) - else: - result = await invoke_porter_action(message, client, config) - assert result is expected_success - - assert not parameters.exists() - assert not installation.exists() - assert command.await_args_list[-3].args[0] == action - assert command.await_args_list[-2].args[0] == ["porter", "parameters", "delete", "test-parameters"] - assert command.await_args_list[-1].args[0] == [ - "porter", - "installations", - "list", - "--name", - "resource", - "--output", - "json", - "--verbosity", - "error", - ] - assert command.await_args_list[-1].kwargs["log_output"] is False - outputs.assert_not_awaited() - if expected_success is None: - assert sender.send_messages.await_count == 1 - else: - assert sender.send_messages.await_count == 2 - final_status = json.loads(str(sender.send_messages.call_args.args[0])) - assert final_status["status"] == ("deleted" if expected_success else "deleting_failed") - assert "outputs" not in final_status - if expected_success: - assert "Porter confirmed that the installation is absent" in final_status["message"] - else: - assert error in final_status["message"] - - @pytest.mark.asyncio @patch("vmss_porter.runner.run_command_helper") async def test_run_porter_azure_login_failure(mock_run_command_helper): diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index e11154cb07..3d731fd9bb 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -325,34 +325,6 @@ async def _cleanup_param_set(param_set_name: str, param_set_file: str, installat logger.debug(f"Best-effort cleanup: could not delete temp file '{path}': {e}") -async def _confirm_porter_installation_absent(installation_id: str, config: dict) -> bool: - """Confirm absence with a fresh storage query, without logging installation data.""" - try: - code, output, _ = await asyncio.wait_for( - run_command_helper( - [ - "porter", - "installations", - "list", - "--name", - installation_id, - "--output", - "json", - "--verbosity", - "error", - ], - config, - "Confirm missing installation", - log_output=False, - ), - timeout=30, - ) - return code == 0 and json.loads(output) == [] - except Exception: - # An unreadable store, invalid response or timeout does not prove absence. - return False - - async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, config: dict) -> bool: """ Handle resource message by invoking specified porter action (i.e. install, uninstall) @@ -405,35 +377,33 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf logger.debug(f"Best-effort cleanup failed for '{installation_id}': {e}") logger.debug("Finished running porter execution command.") - action_completed_without_error = returncode == 0 + action_completed_without_error = False - # Handle command output - if ( - returncode != 0 - and action == "uninstall" - and "could not find installation" in (err or "") - and await _confirm_porter_installation_absent(installation_id, config) - ): - # The error text only triggers the check. A successful empty query is - # required before completing deletion without installation outputs. + if returncode == 0: action_completed_without_error = True - status_for_sb_message = statuses.pass_status_string_for[action] - resource_request_message = service_bus_message_generator( - msg_body, status_for_sb_message, "Uninstall completed: Porter confirmed that the installation is absent." - ) - elif returncode != 0: + + # Handle command output + if returncode != 0 and err is not None: # Construct a readable representation of the command(s) command_representation = "" for cmd in porter_command: command_representation += " ".join(cmd) + "; " command_representation = command_representation.rstrip("; ") - error_output = err or f"Porter exited with code {returncode} and no error output." - error_message = ( - "Error message: " + " ".join(error_output.split("\n")) + "; Porter command: " + command_representation - ) - # Installation lookup errors can wrap storage failures. They do not prove that resources were deleted. - status_for_sb_message = statuses.failed_status_string_for[action] + error_message = "Error message: " + " ".join(err.split("\n")) + "; Porter command: " + command_representation + action_completed_without_error = False + + if "uninstall" == action and "could not find installation" in err: + logger.warning( + "The installation doesn't exist. Treating as a successful action to allow the flow to proceed." + ) + action_completed_without_error = True + error_message = f"A success despite of underlying error. {error_message}" + + if action_completed_without_error: + status_for_sb_message = statuses.pass_status_string_for[action] + else: + status_for_sb_message = statuses.failed_status_string_for[action] resource_request_message = service_bus_message_generator(msg_body, status_for_sb_message, error_message) diff --git a/templates/shared_services/airlock_notifier/Dockerfile.tmpl b/templates/shared_services/airlock_notifier/Dockerfile.tmpl index a06203172f..816a6fb388 100644 --- a/templates/shared_services/airlock_notifier/Dockerfile.tmpl +++ b/templates/shared_services/airlock_notifier/Dockerfile.tmpl @@ -14,11 +14,6 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ # PORTER_MIXINS -# Fail the build if the bundled CLI cannot select a user-assigned identity. -RUN az login --help > /tmp/azure-login-help && \ - grep -q -- '--client-id' /tmp/azure-login-help && \ - rm /tmp/azure-login-help - # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/shared_services/airlock_notifier/porter.yaml b/templates/shared_services/airlock_notifier/porter.yaml index 40e1a583de..87274084b3 100644 --- a/templates/shared_services/airlock_notifier/porter.yaml +++ b/templates/shared_services/airlock_notifier/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-airlock-notifier -version: 1.0.14 +version: 1.0.13 description: "A shared service notifying on Airlock Operations" registry: azuretre dockerfile: Dockerfile.tmpl @@ -75,7 +75,7 @@ parameters: mixins: - exec - az: - clientVersion: 2.81.0 + clientVersion: 2.67.0 - terraform: clientVersion: 1.9.8 diff --git a/templates/shared_services/certs/Dockerfile.tmpl b/templates/shared_services/certs/Dockerfile.tmpl index 3084917f6b..f61b684795 100644 --- a/templates/shared_services/certs/Dockerfile.tmpl +++ b/templates/shared_services/certs/Dockerfile.tmpl @@ -10,7 +10,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa # Install jq + git (git is required for terraform_azurerm_environment_configuration) RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ apt-get update \ - && apt-get install -y --no-install-recommends git jq openssl ca-certificates + && apt-get install -y --no-install-recommends git jq # Install Certbot RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 0cbb66a7a9..2ce29f568c 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-certs -version: 0.7.16 +version: 0.7.13 description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" registry: azuretre dockerfile: Dockerfile.tmpl @@ -67,14 +67,6 @@ mixins: clientVersion: 2.81.0 install: - - exec: - description: "Validate the certificate name" - command: python3 - arguments: - - ./scripts/certificate_cache.py - - validate-name - flags: - cert-name: ${ bundle.parameters.cert_name } - terraform: description: "Deploy shared service" vars: diff --git a/templates/shared_services/certs/scripts/certificate_cache.py b/templates/shared_services/certs/scripts/certificate_cache.py deleted file mode 100644 index 5a996a9dcf..0000000000 --- a/templates/shared_services/certs/scripts/certificate_cache.py +++ /dev/null @@ -1,232 +0,0 @@ -#!/usr/bin/env python3 -"""Retain verified certificates in the core vault across service replacement.""" - -import argparse -import hashlib -import json -import os -from pathlib import Path -import re -import subprocess -import sys -from datetime import datetime, timezone - -CACHE_PREFIX = "tre-acme-" -CACHE_TAG = "certificate-v1" -RENEW_BEFORE_SECONDS = 30 * 24 * 60 * 60 -CACHE_MISS = 3 - - -class CertificateNotFound(RuntimeError): - """The vault returned its explicit certificate-not-found response.""" - - -def cache_name(fqdn): - return CACHE_PREFIX + hashlib.sha256(fqdn.lower().encode("ascii")).hexdigest() - - -def run(arguments): - """Do not expose command output or arguments when a secret operation fails.""" - try: - return subprocess.run(arguments, capture_output=True, text=True, timeout=180, check=True).stdout - except subprocess.CalledProcessError as error: - code = re.search(r"^\s*(?:ERROR: )?\(([A-Za-z0-9]+)\)", error.stderr or "", re.MULTILINE) - if code and code[1] == "CertificateNotFound": - raise CertificateNotFound("Certificate not found") from None - raise RuntimeError(f"Certificate command failed ({code[1] if code else 'see operation diagnostics'})") from None - except subprocess.TimeoutExpired: - raise RuntimeError("Certificate command timed out") from None - - -def azure(*arguments): - return json.loads(run(["az", *arguments, "--only-show-errors", "--output", "json"])) - - -def validate_name(name): - if name.lower().startswith(CACHE_PREFIX): - raise ValueError(f"Certificate names starting with {CACHE_PREFIX} are reserved for the retained cache") - - -def validate_material(directory, fqdn, now=None): - """Check exact SAN, public trust, validity and the private key before reuse.""" - leaf = directory / "leaf.pem" - fullchain = directory / "fullchain.pem" - key = directory / "privkey.pem" - run(["openssl", "x509", "-in", str(fullchain), "-out", str(leaf)]) - san = run(["openssl", "x509", "-in", str(leaf), "-noout", "-ext", "subjectAltName"]) - names = re.findall(r"DNS:([^,\s]+)", san) - if fqdn.lower() not in [name.lower() for name in names]: - raise ValueError("Certificate does not contain the exact requested DNS name") - dates = run(["openssl", "x509", "-in", str(leaf), "-noout", "-dates"]) - bounds = dict(line.split("=", 1) for line in dates.strip().splitlines()) - start, end = [ - datetime.strptime(bounds[k], "%b %d %H:%M:%S %Y %Z").replace(tzinfo=timezone.utc) - for k in ("notBefore", "notAfter") - ] - now = now or datetime.now(timezone.utc) - if start > now: - raise ValueError("Certificate is not yet valid") - certificate_key = run(["openssl", "x509", "-in", str(leaf), "-noout", "-pubkey"]) - private_key = run(["openssl", "pkey", "-in", str(key), "-pubout"]) - if certificate_key.strip() != private_key.strip(): - raise ValueError("Certificate and private key do not match") - # Verify the chain at an instant within the certificate validity period. - # Expired but otherwise valid material requests renewal below. - verify_time = now.timestamp() if now < end else end.timestamp() - 1 - run( - [ - "openssl", - "verify", - "-purpose", - "sslserver", - "-verify_hostname", - fqdn, - "-attime", - str(int(verify_time)), - "-untrusted", - str(fullchain), - str(leaf), - ] - ) - return (end - now).total_seconds() > RENEW_BEFORE_SECONDS - - -def restore(vault, fqdn, directory): - name = cache_name(fqdn) - try: - certificate = azure("keyvault", "certificate", "show", "--vault-name", vault, "--name", name) - except CertificateNotFound: - # A deleted cache is an explicit administrative state, not permission to issue again. - try: - azure("keyvault", "certificate", "show-deleted", "--vault-name", vault, "--name", name) - except CertificateNotFound: - print("No retained certificate. Production issuance is required.") - return False - raise RuntimeError("The retained certificate is deleted. Recover it before retrying") - if certificate.get("attributes", {}).get("enabled") is not True: - raise ValueError("The retained certificate is disabled") - if certificate.get("tags", {}).get("tre-cache") != CACHE_TAG: - raise ValueError("The retained certificate has no recognised ownership tag") - if certificate.get("tags", {}).get("fqdn") != fqdn.lower(): - raise ValueError("The retained certificate belongs to another DNS name") - content_type = certificate.get("policy", {}).get("secretProperties", {}).get("contentType") - if content_type not in ("application/x-pem-file", "application/x-pkcs12"): - raise ValueError("The retained certificate has an unsupported or missing content type") - is_pem = content_type == "application/x-pem-file" - # Download the exact version in its stored format. PEM secrets contain text; - # PFX secrets contain base64 data. Downloading does not convert the format. - material = directory / ("retained.pem" if is_pem else "retained.pfx") - run( - [ - "az", - "keyvault", - "secret", - "download", - "--id", - certificate["sid"], - "--file", - str(material), - "--encoding", - "utf-8" if is_pem else "base64", - "--only-show-errors", - ] - ) - if is_pem: - certificates = re.findall( - r"-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", material.read_text(), re.DOTALL - ) - if not certificates: - raise ValueError("The retained PEM contains no certificates") - (directory / "fullchain.pem").write_text("\n".join(certificates) + "\n") - run(["openssl", "pkey", "-in", str(material), "-out", str(directory / "privkey.pem")]) - else: - run( - [ - "openssl", - "pkcs12", - "-in", - str(material), - "-passin", - "pass:", - "-nokeys", - "-out", - str(directory / "fullchain.pem"), - ] - ) - run( - [ - "openssl", - "pkcs12", - "-in", - str(material), - "-passin", - "pass:", - "-nocerts", - "-nodes", - "-out", - str(directory / "privkey.pem"), - ] - ) - if not validate_material(directory, fqdn): - print("The retained certificate is due for renewal. Production issuance is required.") - return False - print(f"Reusing retained certificate {name}, thumbprint {certificate['x509ThumbprintHex']}") - return True - - -def store(vault, fqdn, directory): - if not validate_material(directory, fqdn): - raise ValueError("The issued certificate is already due for renewal") - # A PEM import includes the private key and complete chain. The CLI reads a private file. - material = directory / "cache-import.pem" - material.write_text((directory / "privkey.pem").read_text() + (directory / "fullchain.pem").read_text()) - try: - certificate = azure( - "keyvault", - "certificate", - "import", - "--vault-name", - vault, - "--name", - cache_name(fqdn), - "--file", - str(material), - "--tags", - f"tre-cache={CACHE_TAG}", - f"fqdn={fqdn.lower()}", - ) - finally: - material.unlink(missing_ok=True) - print(f"Retained issued certificate {cache_name(fqdn)}, thumbprint {certificate['x509ThumbprintHex']}") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("validate-name", "restore", "store")) - parser.add_argument("--cert-name", required=True) - parser.add_argument("--vault") - parser.add_argument("--fqdn") - parser.add_argument("--directory", type=Path) - args = parser.parse_args() - os.umask(0o077) - try: - validate_name(args.cert_name) - if args.action == "validate-name": - return 0 - if not args.vault or not args.fqdn or not args.directory: - parser.error("--vault, --fqdn and --directory are required") - if args.action == "restore": - return 0 if restore(args.vault, args.fqdn, args.directory) else CACHE_MISS - store(args.vault, args.fqdn, args.directory) - return 0 - except (ValueError, KeyError, OSError, RuntimeError): - # Exception arguments can contain secret command results or certificate material. - print( - "Certificate reuse failed. Check vault access, cache ownership, validity and trust before retrying.", - file=sys.stderr, - ) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/templates/shared_services/certs/scripts/letsencrypt.sh b/templates/shared_services/certs/scripts/letsencrypt.sh index 110a56301c..bd88397215 100755 --- a/templates/shared_services/certs/scripts/letsencrypt.sh +++ b/templates/shared_services/certs/scripts/letsencrypt.sh @@ -1,7 +1,5 @@ #!/bin/bash set -e -set -o pipefail -umask 077 script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") @@ -52,88 +50,71 @@ done # done with processing args and can set this set -o nounset -work_dir=$(mktemp -d) -trap 'rm -rf "$work_dir"' EXIT -cache_args=(--cert-name "$cert_name" --vault "$keyvault_name" --fqdn "$fqdn" --directory "$work_dir") +echo "Checking for index.html file in storage account" -if python3 "${script_dir}/certificate_cache.py" restore "${cache_args[@]}"; then - echo "Using the retained certificate without an ACME request" -else - cache_status=$? - if [[ "$cache_status" -ne 3 ]]; then - exit "$cache_status" - fi - echo "Checking for index.html file in storage account" - - # Create the default index.html page - cat << EOF > "${work_dir}/index.html" +# Create the default index.html page +cat << EOF > index.html EOF +# shellcheck disable=SC2016 +indexExists=$(az storage blob list -o json \ + --account-name "${storage_account_name}" \ + --auth-mode login \ + --container-name '$web' \ + --query "[?name=='index.html'].name" \ + | jq 'length') + +if [[ ${indexExists} -lt 1 ]]; then + echo "No existing file found. Uploading index.html file" + # shellcheck disable=SC2016 - indexExists=$(az storage blob list -o json \ + az storage blob upload \ --account-name "${storage_account_name}" \ --auth-mode login \ --container-name '$web' \ - --query "[?name=='index.html'].name" \ - | jq 'length') - - if [[ ${indexExists} -lt 1 ]]; then - echo "No existing file found. Uploading index.html file" - - # shellcheck disable=SC2016 - az storage blob upload \ - --account-name "${storage_account_name}" \ - --auth-mode login \ - --container-name '$web' \ - --file "${work_dir}/index.html" \ - --name index.html \ - --no-progress \ - --only-show-errors - - # Wait a bit for the App Gateway health probe to notice - echo "Waiting 30s for app gateway health probe" - sleep 30s - else - echo "index.html already present" - fi - - ledir="${work_dir}/letsencrypt" - mkdir -p "${ledir}/logs" - - # Initiate the ACME challenge - echo "Initiating ACME challenge" - export STORAGE_ACCOUNT_NAME="${storage_account_name}" - /opt/certbot/bin/certbot certonly \ - --config-dir "${ledir}" \ - --work-dir "${ledir}" \ - --logs-dir "${ledir}"/logs \ - --manual \ - --preferred-challenges=http \ - --manual-auth-hook "${script_dir}"/auth-hook.sh \ - --manual-cleanup-hook "${script_dir}"/cleanup-hook.sh \ - --domain "${fqdn}" \ - --non-interactive \ - --agree-tos \ - --register-unsafely-without-email - - cp "${ledir}/live/${fqdn}/privkey.pem" "${work_dir}/privkey.pem" - cp "${ledir}/live/${fqdn}/fullchain.pem" "${work_dir}/fullchain.pem" - # Persist issuance before updating the service alias. A later failure can reuse it. - python3 "${script_dir}/certificate_cache.py" store "${cache_args[@]}" + --file index.html \ + --name index.html \ + --no-progress \ + --only-show-errors + + # Wait a bit for the App Gateway health probe to notice + echo "Waiting 30s for app gateway health probe" + sleep 30s +else + echo "index.html already present" fi +ledir="${script_dir}/../letsencrypt" +mkdir -p "${ledir}/logs" + +# Initiate the ACME challange +echo "Initiating ACME challenge" +export STORAGE_ACCOUNT_NAME="${storage_account_name}" +/opt/certbot/bin/certbot certonly \ + --config-dir "${ledir}" \ + --work-dir "${ledir}" \ + --logs-dir "${ledir}"/logs \ + --manual \ + --preferred-challenges=http \ + --manual-auth-hook "${script_dir}"/auth-hook.sh \ + --manual-cleanup-hook "${script_dir}"/cleanup-hook.sh \ + --domain "${fqdn}" \ + --non-interactive \ + --agree-tos \ + --register-unsafely-without-email + + # Convert the generated certificate to a .pfx echo "Got cert. Converting to PFX" -CERT_DIR="${work_dir}" +CERT_DIR="${ledir}/live/${fqdn}" CERT_PASSWORD=$(openssl rand -base64 30) -printf %s "$CERT_PASSWORD" > "${CERT_DIR}/password" openssl pkcs12 -export \ -inkey "${CERT_DIR}/privkey.pem" \ -in "${CERT_DIR}/fullchain.pem" \ -out "${CERT_DIR}/aci.pfx" \ - -passout "file:${CERT_DIR}/password" + -passout "pass:${CERT_PASSWORD}" # Save cert and password to KeyVault echo "Importing cert to KeyVault ${keyvault_name}" @@ -148,8 +129,7 @@ sid=$(az keyvault certificate import \ echo "Saving certificate password to KV with key ${password_name}" az keyvault secret set --name "$password_name" \ --vault-name "${keyvault_name}" \ - --file "${CERT_DIR}/password" \ - --output none --only-show-errors + --value "${CERT_PASSWORD}" echo "Updating SSL cert in app gateway" az network application-gateway ssl-cert update \ diff --git a/templates/shared_services/certs/scripts/wait_for_private_storage.py b/templates/shared_services/certs/scripts/wait_for_private_storage.py deleted file mode 100644 index c357930d99..0000000000 --- a/templates/shared_services/certs/scripts/wait_for_private_storage.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Wait for consecutive private DNS and verified TLS checks before storage writes.""" - -import argparse -import ipaddress -import json -import os -from pathlib import Path -import socket -import ssl -import subprocess -import sys -import time - - -PROBE_TIMEOUT_SECONDS = 10 -TIMEOUT_SECONDS = 300 -REQUIRED_SUCCESSES = 7 -INTERVAL_SECONDS = 5 - - -def probe(hostname, expected_ip): - """Connect only when every DNS answer matches the private endpoint address.""" - try: - answers = { - ipaddress.ip_address(item[4][0]) for item in socket.getaddrinfo(hostname, 443, type=socket.SOCK_STREAM) - } - if answers != {ipaddress.ip_address(expected_ip)}: - return False, "DNS does not exclusively resolve to the expected private endpoint" - context = ssl.create_default_context() - context.minimum_version = ssl.TLSVersion.TLSv1_2 - with socket.create_connection((expected_ip, 443), timeout=5) as connection: - with context.wrap_socket(connection, server_hostname=hostname): - return True, "private DNS and TLS verification passed" - except (OSError, ValueError) as error: - return False, f"DNS or TLS check failed ({type(error).__name__})" - - -def run_probe(hostname, expected_ip, timeout): - # A child process bounds getaddrinfo(), which has no per-call timeout. - try: - result = subprocess.run( - [ - sys.executable, - str(Path(__file__).resolve()), - "--probe", - "--hostname", - hostname, - "--expected-ip", - expected_ip, - ], - capture_output=True, - text=True, - timeout=timeout, - check=False, - ) - except subprocess.TimeoutExpired: - return False, "DNS or TLS check timed out" - if result.returncode not in (0, 1): - return False, "DNS or TLS check process failed" - try: - value = json.loads(result.stdout) - if type(value.get("ready")) is not bool or not isinstance(value.get("reason"), str): - raise ValueError("Invalid probe result") - except (AttributeError, ValueError): - return False, "DNS or TLS check returned an invalid result" - if value["ready"] != (result.returncode == 0): - return False, "DNS or TLS check returned an inconsistent result" - return value["ready"], value["reason"] - - -def wait_for_private_storage( - hostname, expected_ip, timeout=TIMEOUT_SECONDS, required_successes=REQUIRED_SUCCESSES, interval=INTERVAL_SECONDS -): - deadline = time.monotonic() + timeout - successes = 0 - last_reason = "no completed check" - while time.monotonic() < deadline: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - ready, last_reason = run_probe(hostname, expected_ip, min(PROBE_TIMEOUT_SECONDS, remaining)) - if time.monotonic() >= deadline: - break - successes = successes + 1 if ready else 0 - print(f"Storage readiness: {last_reason} ({successes}/{required_successes} consecutive checks)", flush=True) - if successes >= required_successes: - return - time.sleep(min(interval, max(0, deadline - time.monotonic()))) - raise TimeoutError( - f"Private storage endpoint was not ready within {timeout} seconds: {last_reason}. Check private DNS and endpoint connectivity before retrying." - ) - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--hostname", default=os.environ.get("STORAGE_BLOB_HOST")) - parser.add_argument("--expected-ip", default=os.environ.get("STORAGE_PRIVATE_IP")) - parser.add_argument("--probe", action="store_true", help=argparse.SUPPRESS) - args = parser.parse_args() - if not args.hostname or any(c.isspace() for c in args.hostname) or any(c in args.hostname for c in "/:\\"): - parser.error("A storage DNS hostname is required") - try: - address = ipaddress.ip_address(args.expected_ip) - except ValueError: - parser.error("A private endpoint IP address is required") - if ( - not address.is_private - or address.is_loopback - or address.is_link_local - or address.is_unspecified - or address.is_multicast - ): - parser.error("The expected address must be a private endpoint IP address") - return args - - -def main(): - args = parse_args() - if args.probe: - ready, reason = probe(args.hostname, args.expected_ip) - print(json.dumps({"ready": ready, "reason": reason})) - return 0 if ready else 1 - try: - wait_for_private_storage(args.hostname, args.expected_ip) - except (OSError, TimeoutError) as error: - print(f"Error: {error}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/templates/shared_services/certs/terraform/staticweb.tf b/templates/shared_services/certs/terraform/staticweb.tf index 1d5b97c3a0..ee8093ed53 100644 --- a/templates/shared_services/certs/terraform/staticweb.tf +++ b/templates/shared_services/certs/terraform/staticweb.tf @@ -47,30 +47,7 @@ resource "azurerm_storage_account_static_website" "staticweb_site" { index_document = "index.html" error_404_document = "404.html" - depends_on = [terraform_data.staticweb_private_endpoint_ready] -} - -# Endpoint creation can finish before the DNS answers become stable. -resource "terraform_data" "staticweb_private_endpoint_ready" { - triggers_replace = { - endpoint = azurerm_private_endpoint.blobpe.id - address = azurerm_private_endpoint.blobpe.private_service_connection[0].private_ip_address - helper = filesha256("${path.module}/../scripts/wait_for_private_storage.py") - } - - lifecycle { - replace_triggered_by = [azurerm_private_endpoint.blobpe] - } - - depends_on = [azurerm_private_endpoint.webpe] - - provisioner "local-exec" { - command = "python3 \"${path.module}/../scripts/wait_for_private_storage.py\"" - environment = { - STORAGE_BLOB_HOST = azurerm_storage_account.staticweb.primary_blob_host - STORAGE_PRIVATE_IP = azurerm_private_endpoint.blobpe.private_service_connection[0].private_ip_address - } - } + depends_on = [azurerm_private_endpoint.blobpe, azurerm_private_endpoint.webpe] } resource "azurerm_role_assignment" "stgwriter" { diff --git a/templates/shared_services/cyclecloud/Dockerfile.tmpl b/templates/shared_services/cyclecloud/Dockerfile.tmpl index 1b9c510f9a..abc241c3a1 100644 --- a/templates/shared_services/cyclecloud/Dockerfile.tmpl +++ b/templates/shared_services/cyclecloud/Dockerfile.tmpl @@ -11,10 +11,5 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ # PORTER_MIXINS -# Fail the build if the bundled CLI cannot select a user-assigned identity. -RUN az login --help > /tmp/azure-login-help && \ - grep -q -- '--client-id' /tmp/azure-login-help && \ - rm /tmp/azure-login-help - # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/shared_services/cyclecloud/porter.yaml b/templates/shared_services/cyclecloud/porter.yaml index 1cd58af1d4..a65f7d24ec 100644 --- a/templates/shared_services/cyclecloud/porter.yaml +++ b/templates/shared_services/cyclecloud/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-cyclecloud -version: 0.7.9 +version: 0.7.8 description: "An Azure TRE Shared Service Template for Azure Cyclecloud" registry: azuretre dockerfile: Dockerfile.tmpl @@ -71,7 +71,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.81.0 + clientVersion: 2.67.0 install: - terraform: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl index d824805e29..4a0d5cc05c 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/Dockerfile.tmpl @@ -29,11 +29,6 @@ COPY --from=import-reviewvm ./terraform/download_review_data.ps1 ${BUNDLE_DIR}/t # PORTER_MIXINS -# Fail the build if the bundled CLI cannot select a user-assigned identity. -RUN az login --help > /tmp/azure-login-help && \ - grep -q -- '--client-id' /tmp/azure-login-help && \ - rm /tmp/azure-login-help - # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml index 5aae3ab735..632094c4ac 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-export-reviewvm -version: 2.0.9 +version: 2.0.6 description: "An Azure TRE User Resource Template for reviewing Airlock export requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -9,10 +9,10 @@ registry: azuretre custom: # For information on vm_sizes and image_options, see README.me in the guacamole/user-resources folder vm_sizes: - "2 CPU | 8GB RAM": Standard_D2s_v6 - "4 CPU | 16GB RAM": Standard_D4s_v6 - "8 CPU | 32GB RAM": Standard_D8s_v6 - "16 CPU | 64GB RAM": Standard_D16s_v6 + "2 CPU | 8GB RAM": Standard_D2s_v7 + "4 CPU | 16GB RAM": Standard_D4s_v7 + "8 CPU | 32GB RAM": Standard_D8s_v7 + "16 CPU | 64GB RAM": Standard_D16s_v7 image_options: "Windows Server 2025": source_image_reference: @@ -122,7 +122,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.81.0 + clientVersion: 2.67.0 install: - terraform: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl index 2850da0648..c77706ca7a 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/Dockerfile.tmpl @@ -14,11 +14,6 @@ COPY --from=windowsvm ./terraform/vm ${BUNDLE_DIR}/terraform/vm # PORTER_MIXINS -# Fail the build if the bundled CLI cannot select a user-assigned identity. -RUN az login --help > /tmp/azure-login-help && \ - grep -q -- '--client-id' /tmp/azure-login-help && \ - rm /tmp/azure-login-help - # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml index 36c78cdc4f..1e06fcaca2 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-import-reviewvm -version: 2.0.6 +version: 2.0.3 description: "An Azure TRE User Resource Template for reviewing Airlock import requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -9,10 +9,10 @@ registry: azuretre custom: # For information on vm_sizes and image_options, see README.me in the guacamole/user-resources folder vm_sizes: - "2 CPU | 8GB RAM": Standard_D2s_v6 - "4 CPU | 16GB RAM": Standard_D4s_v6 - "8 CPU | 32GB RAM": Standard_D8s_v6 - "16 CPU | 64GB RAM": Standard_D16s_v6 + "2 CPU | 8GB RAM": Standard_D2s_v7 + "4 CPU | 16GB RAM": Standard_D4s_v7 + "8 CPU | 32GB RAM": Standard_D8s_v7 + "16 CPU | 64GB RAM": Standard_D16s_v7 image_options: "Windows Server 2025": source_image_reference: @@ -129,7 +129,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.81.0 + clientVersion: 2.67.0 install: - terraform: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf index 389e7027df..074ac3b848 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf @@ -28,7 +28,6 @@ module "windows_vm" { nexus_proxy_url = local.nexus_proxy_url install_azure_cli = true - require_azure_cli = true install_vscode = true install_storage_explorer = true install_git = false diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 7bf342ff5c..dfc9e326ff 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-linuxvm -version: 1.4.9 +version: 1.4.8 description: "An Azure TRE User Resource Template for Guacamole (Linux)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -9,10 +9,10 @@ registry: azuretre custom: # For information on vm_sizes and image_options, see README.me in the guacamole/user-resources folder vm_sizes: - "2 CPU | 8GB RAM": Standard_D2s_v6 - "4 CPU | 16GB RAM": Standard_D4s_v6 - "8 CPU | 32GB RAM": Standard_D8s_v6 - "16 CPU | 64GB RAM": Standard_D16s_v6 + "2 CPU | 8GB RAM": Standard_D2s_v7 + "4 CPU | 16GB RAM": Standard_D4s_v7 + "8 CPU | 32GB RAM": Standard_D8s_v7 + "16 CPU | 64GB RAM": Standard_D16s_v7 image_options: "Ubuntu 22.04 LTS": source_image_reference: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index cb7aef8d6f..e7b2686f5c 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-windowsvm -version: 3.0.5 +version: 3.0.4 description: "An Azure TRE User Resource Template for Guacamole (Windows 11 or Windows Server 2025)" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf index 2298b896b7..4e653f2f66 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/locals.tf @@ -36,7 +36,6 @@ locals { nexus_proxy_url = var.nexus_proxy_url SharedStorageAccess = var.shared_storage_access ? 1 : 0 InstallAzureCli = var.install_azure_cli ? 1 : 0 - RequireAzureCli = var.require_azure_cli ? 1 : 0 InstallVsCode = var.install_vscode ? 1 : 0 InstallStorageExplorer = var.install_storage_explorer ? 1 : 0 InstallGit = var.install_git ? 1 : 0 diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf index 8dfc640804..581ebb3d57 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/variables.tf @@ -47,11 +47,6 @@ variable "install_azure_cli" { type = bool default = true } -variable "require_azure_cli" { - description = "Fail bootstrap when Nexus or Azure CLI is unavailable, as required by Airlock review VMs" - type = bool - default = false -} variable "install_vscode" { type = bool default = true diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 index 725f968d96..305dc1bdbe 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm/vm_config.ps1 @@ -28,7 +28,7 @@ trusted-host = $NexusHost $Utf8NoBomEncoding = New-Object System.Text.UTF8Encoding $False [System.IO.File]::WriteAllLines($PipConfigFilePath, $ConfigBody, $Utf8NoBomEncoding) -# Docker proxy config (best-effort; only applies if Docker is pre-installed) +# Docker proxy config (best-effort — only applies if Docker is pre-installed) try { $dockerConfigDir = "$env:ProgramData\docker\config" if (!(Test-Path $dockerConfigDir)) { @@ -55,7 +55,6 @@ $RStudioVersion = "2026.07.1-147" $GitVersion = "2.55.0" $MiniforgePath = "C:\Miniforge3" $InstallAzureCli = (${InstallAzureCli} -eq 1) -$RequireAzureCli = (${RequireAzureCli} -eq 1) $InstallVsCode = (${InstallVsCode} -eq 1) $InstallStorageExplorer = (${InstallStorageExplorer} -eq 1) $InstallGit = (${InstallGit} -eq 1) @@ -104,11 +103,10 @@ function Install-TreTool { [Parameter(Mandatory = $true)][string]$OutFile, [Parameter(Mandatory = $true)][string]$FilePath, [Parameter(Mandatory = $true)][string[]]$ArgumentList, - [int]$MaxAttempts = 5, - [switch]$Required + [int]$MaxAttempts = 5 ) - # Optional tools may fail without aborting the VM configuration. + # Skip installs on failure rather than aborting the whole VM configuration. try { Write-Host "Downloading $Name from $Url" for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) { @@ -124,13 +122,11 @@ function Install-TreTool { } Write-Host "Installing $Name" $process = Start-Process -FilePath $FilePath -ArgumentList $ArgumentList -Wait -PassThru -NoNewWindow - if ($process.ExitCode -notin @(0, 3010)) { - if ($Required) { throw "$Name installer exited with code $($process.ExitCode)" } + if ($process.ExitCode -ne 0) { Write-Host "WARNING: $Name installer exited with code $($process.ExitCode)" } } catch { - if ($Required) { throw "Required tool $Name could not be installed: $($_.Exception.Message)" } Write-Host "WARNING: Failed to install $Name - $($_.Exception.Message)" } } @@ -209,10 +205,7 @@ function New-DesktopShortcut { # Make sure the proxy is up before pulling any of the installers below. if ($NexusActionsEnabled) { - $NexusReady = Wait-ForNexus - if ($RequireAzureCli -and -not $NexusReady) { - throw "Airlock review requires a reachable Nexus proxy. Check Nexus deployment, DNS and HTTPS connectivity." - } + Wait-ForNexus | Out-Null } else { Write-Host "No Nexus-backed install/config actions enabled - skipping the wait for the Nexus proxy." @@ -228,35 +221,14 @@ if ($ConfigureConda) Configure-RProxy $AzureCliMsi = Join-Path $ToolsDir "azure-cli.msi" -if ($RequireAzureCli -and -not $InstallAzureCli) { - throw "Airlock review requires Azure CLI installation to be enabled." -} if ($InstallAzureCli) { - if ($RequireAzureCli) { - # Probe the required package inside the workspace network before installing tools. - try { - Invoke-WebRequest -Uri "${nexus_proxy_url}/repository/azure-cli/azure-cli-$AzureCliVersion-x64.msi" ` - -Method Head -UseBasicParsing -TimeoutSec 30 | Out-Null - } - catch { - throw "Required Azure CLI package is unavailable through Nexus: $($_.Exception.Message)" - } - } Install-TreTool -Name "Azure CLI" ` -Url "${nexus_proxy_url}/repository/azure-cli/azure-cli-$AzureCliVersion-x64.msi" ` -OutFile $AzureCliMsi ` -FilePath "msiexec.exe" ` - -ArgumentList @("/i", "`"$AzureCliMsi`"", "/qn", "/norestart") ` - -Required:$RequireAzureCli + -ArgumentList @("/i", "`"$AzureCliMsi`"", "/qn", "/norestart") $env:Path = "$Env:ProgramFiles\Microsoft SDKs\Azure\CLI2\wbin;$Env:ProgramFiles (x86)\Microsoft SDKs\Azure\CLI2\wbin;$env:Path" - if ($RequireAzureCli) { - if (-not (Get-Command az -ErrorAction SilentlyContinue)) { - throw "Required Azure CLI executable was not found after installation." - } - & az version --output json | Out-Null - if ($LASTEXITCODE -ne 0) { throw "Required Azure CLI failed its version check." } - } } $VsCodeSetup = Join-Path $ToolsDir "vscode-setup.exe" diff --git a/templates/workspace_services/ohdsi/Dockerfile.tmpl b/templates/workspace_services/ohdsi/Dockerfile.tmpl index 5a9591d773..b2674f6d8a 100644 --- a/templates/workspace_services/ohdsi/Dockerfile.tmpl +++ b/templates/workspace_services/ohdsi/Dockerfile.tmpl @@ -33,10 +33,5 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/ # PORTER_MIXINS -# Fail the build if the bundled CLI cannot select a user-assigned identity. -RUN az login --help > /tmp/azure-login-help && \ - grep -q -- '--client-id' /tmp/azure-login-help && \ - rm /tmp/azure-login-help - # Use the BUNDLE_DIR build argument to copy files into the bundle COPY --link . ${BUNDLE_DIR}/ diff --git a/templates/workspace_services/ohdsi/porter.yaml b/templates/workspace_services/ohdsi/porter.yaml index dba38adcd1..de2e88aba2 100644 --- a/templates/workspace_services/ohdsi/porter.yaml +++ b/templates/workspace_services/ohdsi/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-service-ohdsi -version: 0.3.13 +version: 0.3.12 description: "An OHDSI workspace service" registry: azuretre dockerfile: Dockerfile.tmpl @@ -70,7 +70,7 @@ mixins: - terraform: clientVersion: 1.9.8 - az: - clientVersion: 2.81.0 + clientVersion: 2.67.0 - exec outputs: From f23259d86eefbee49bc5225442e02831e893abd0 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 14 Oct 2026 13:08:59 +0100 Subject: [PATCH 10/17] Verify certificate reuse after reinstallation (#5245) Compare public gateway certificates across the certificate service lifecycle. Require correlated reuse evidence, verify cleanup, and reject old bundles before provisioning. --- CHANGELOG.md | 2 +- docs/tre-developers/end-to-end-tests.md | 25 +- e2e_tests/bundle_coverage.json | 14 +- e2e_tests/bundle_evidence.py | 17 ++ e2e_tests/resources/certificate.py | 174 +++++++++++++ e2e_tests/test_shared_services.py | 97 +++++-- .../unit_tests/test_certificate_evidence.py | 236 ++++++++++++++++++ .../unit_tests/test_certificate_selection.py | 112 ++++++++- 8 files changed, 637 insertions(+), 40 deletions(-) create mode 100644 e2e_tests/resources/certificate.py create mode 100644 e2e_tests/unit_tests/test_certificate_evidence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fb5e8dc65..ef13f3edee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Setting to `false` will delete existing airlock storage accounts and must only be done once all workspaces use the v2 airlock. ([#5048](https://github.com/microsoft/AzureTRE/pull/5048)) ENHANCEMENTS: -* Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL and Airlock storage. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) +* Select individual bundle tests and record lifecycle evidence for base workspaces, private Azure SQL, Airlock storage and certificate reuse across reinstallation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Enable `user_management_enabled`, `auto_workspace_group_creation`, and `auto_grant_workspace_consent` by default in `config.sample.yaml` for a better out-of-the-box experience ([#5050](https://github.com/microsoft/AzureTRE/issues/5050)) * Add shared repository instructions and TRE-prefixed component skills for agent-assisted development, security review, validation and troubleshooting. ([#5111](https://github.com/microsoft/AzureTRE/issues/5111)) * Allow numeric CIDR masks in `address_space_size` (e.g. "23") when requesting auto-assigned address spaces; accepts numeric strings and validates the mask range. (`API` 0.28.0, `tre-workspace-base` 2.12.0, `tre-workspace-unrestricted` 0.15.0, `tre-workspace-airlock-import-review` 0.18.0) ([#4733](https://github.com/microsoft/AzureTRE/issues/4733)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 381a8279f9..7b9fd324c1 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -84,8 +84,21 @@ Check the selected entry's prerequisites and limitations before running it. ### Certificate and Nexus selection -Select `tre-shared-service-certs` to create and delete only the certificate shared service. -The case checks the deployed API record and confirms HTTP 404 after deletion. It does not require Nexus EULA consent. +Select `tre-shared-service-certs` to install, uninstall and reinstall the certificate shared service with the same hostname and certificate name. +The case requires certificate bundle version `0.9.0` or later and pins both installations to the selected template version. +It checks the deployed API record, the gateway's public certificate SAN and validity, and the gateway's stopped state. +Reinstallation must preserve the SHA-256 certificate fingerprint and produce a matching resource processor reuse log without issuance evidence. +After each uninstall, the case requires API HTTP 404 and ARM confirmation that the gateway, public IP and challenge storage are absent. +The bundle report records public metadata, reuse and cleanup results. It does not require Nexus EULA consent. + +The evidence helper currently supports `AzureCloud`. It uses the existing deployment identity for ARM and Log Analytics reads. +It checks log access before resource changes. It does not read private keys, open vault network access or add permissions. +Only aggregate log counts enter the test process. Missing, partial or delayed evidence beyond the deadline fails the case. +See [Application Gateway public certificate metadata](https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.management.network.models.applicationgatewaysslcertificate.publiccertdata) +and [Azure Monitor query authentication](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/api/access-api). + +Each installation has a 45-minute provisioning deadline. Cleanup has its own 60-minute deadline. +The complete case has a 285-minute limit, leaving 15 minutes within the five-hour bundle job for setup and reporting. It refuses to change resources when Nexus is present. Existing certificate resources must belong to earlier E2E tests before recovery can remove them. Cleanup has a separate deadline and retains the certificate service if Nexus appears before deletion. Use an isolated environment and run mutating suites sequentially. These guards do not provide a distributed lock. @@ -99,7 +112,13 @@ A skipped case remains unproven coverage. The certificate-only case is opt-in through bundle selection or the `certificate_validation` marker. It is excluded from the ordinary `shared_services` selection to avoid an additional certificate request in that suite. The existing combined case remains in that suite. It deletes Nexus before its certificate dependency. -Certificate content, renewal, retained Key Vault artefacts, Nexus package/proxy use, upgrades and physical resource removal need separate evidence. +Certificate chain validation, actual renewal, retained Key Vault artefacts, Nexus package/proxy use, upgrades and other Azure resource removal need separate evidence. +The case does not purge certificates. A first installation can reuse a certificate retained by an earlier run. + +Before merging the certificate fix, run this case from a trusted upstream branch that contains both the certificate implementation and these tests. +Select `e2eBundle=tre-shared-service-certs` in `deploy_tre_branch.yml` and leave the marker selector empty. +PR slash commands use workflow definitions from `main`. `/test` runs smoke tests and `/test-shared-services` excludes this certificate-only case. +A test-only PR without the new certificate bundle cannot validate reuse. Run Nexus and Airlock only after this case and its cleanup pass. ### Base workspace lifecycle diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index f3a1788d49..23c07e3fff 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -52,14 +52,20 @@ ], "coverage": [ "install", - "deployment-status", + "api-configuration", + "certificate-hostname-and-validity", + "certificate-reuse-after-reinstall", + "gateway-stopped", "uninstall", - "api-removal" + "api-removal", + "gateway-ip-storage-removal" ], "limitations": [ - "Requires the deployed firewall. Uses an isolated environment and refuses to start while Nexus exists. Only earlier E2E certificate resources can be recovered.", + "Requires tre-shared-service-certs 0.9.0 or later, the deployed firewall and an isolated AzureCloud environment. Refuses to start while Nexus exists. Only earlier E2E certificate resources can be recovered.", + "Uses the existing deployment identity for ARM and Log Analytics reads. Requires AppTraces and read access. Does not read Key Vault secrets or change vault network rules or permissions.", + "Compares the public gateway certificate fingerprint and hostname across uninstall and reinstall. Requires a matching resource processor reuse log and rejects issuance evidence. Missing or partial log evidence fails the case.", + "Checks API removal and absence of the gateway, public IP and challenge storage. Does not enumerate all dependent Azure resources or inspect vault soft-deletion, secret values, certificate chains, renewal or upgrades.", "Skips at weekends unless explicitly overridden after checking certificate rate limits. A skip does not prove the lifecycle.", - "Validates the API lifecycle, not certificate contents, renewal, retained Key Vault artefacts, upgrades or physical resource removal.", "Certificate cleanup is retained if Nexus is present. Run mutating suites sequentially; the guard is not a distributed lock." ], "unavailable_reason": "" diff --git a/e2e_tests/bundle_evidence.py b/e2e_tests/bundle_evidence.py index ea44176e43..27f6a2deb5 100644 --- a/e2e_tests/bundle_evidence.py +++ b/e2e_tests/bundle_evidence.py @@ -47,6 +47,22 @@ def record_operation(endpoint, state, done): _ACTIVE_REPORT.write() +def record_certificate_check(phase, resource_path, metadata, *, reused=False, removed=False): + """Retain public certificate evidence, never raw Azure responses or logs.""" + if _ACTIVE_REPORT is None: + return + _ACTIVE_REPORT.data["certificate_checks"].append( + { + "phase": phase, + "resource_path": resource_path, + **{key: metadata[key] for key in ("hostname", "sha256", "gateway_stopped") if key in metadata}, + "reuse_confirmed": reused, + "api_gateway_ip_storage_removed": removed, + } + ) + _ACTIVE_REPORT.write() + + class BundleReport: def __init__(self, filename, data): self.filename = Path(filename) @@ -58,6 +74,7 @@ def __init__(self, filename, data): "resources": [], "reused_resources": [], "operations": {}, + "certificate_checks": [], } self.expected_tests = set(data.get("requested_tests", [])) self.write() diff --git a/e2e_tests/resources/certificate.py b/e2e_tests/resources/certificate.py new file mode 100644 index 0000000000..5d26fa9281 --- /dev/null +++ b/e2e_tests/resources/certificate.py @@ -0,0 +1,174 @@ +"""Read public certificate and cleanup evidence without accessing vault secrets.""" + +import asyncio +import base64 +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +import os +import re +from uuid import UUID + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes + +from e2e_tests import config +from e2e_tests.cloud import get_cloud +from e2e_tests.resources.sql_probe import arm_client + + +NETWORK_API = "2024-05-01" +STORAGE_API = "2023-05-01" +LOGS_API = "2023-09-01" +CHECK_TIMEOUT_SECONDS = 5 * 60 + + +def certificate_metadata(gateway, public_ip, group, tre_id, resource_id): + """Return only public metadata after checking resource ownership and identity.""" + expected_tags = {"tre_id": tre_id, "tre_shared_service_id": resource_id} + for resource, suffix in ( + (gateway, f"Microsoft.Network/applicationGateways/agw-certs-{tre_id}"), + (public_ip, f"Microsoft.Network/publicIPAddresses/pip-cert-nexus-{tre_id}"), + ): + assert resource.get("id", "").lower() == f"{group}/providers/{suffix}".lower(), ( + "Unexpected certificate resource" + ) + assert all(resource.get("tags", {}).get(k) == v for k, v in expected_tags.items()), ( + "Certificate infrastructure does not belong to this test-created service" + ) + fqdn = public_ip["properties"]["dnsSettings"]["fqdn"] + certificates = [c for c in gateway["properties"].get("sslCertificates", []) if c["name"] == "cert-primary"] + assert len(certificates) == 1, "The gateway must have one primary certificate" + properties = certificates[0]["properties"] + secret_id = properties.get("keyVaultSecretId", "") + assert re.fullmatch( + rf"https://kv-{re.escape(tre_id)}\.vault\.azure\.net/secrets/nexus-ssl/[a-fA-F0-9]+", secret_id + ), "The gateway certificate must reference the named certificate in the core vault" + data = base64.b64decode(properties["publicCertData"], validate=True) + cert = ( + x509.load_pem_x509_certificate(data) + if data.startswith(b"-----BEGIN CERTIFICATE-----") + else x509.load_der_x509_certificate(data) + ) + names = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + assert fqdn in names.get_values_for_type(x509.DNSName), "Certificate SAN does not match the service hostname" + now = datetime.now(timezone.utc) + assert cert.not_valid_before_utc <= now, "Certificate is not valid yet" + assert cert.not_valid_after_utc > now + timedelta(days=30), "Certificate is already due for renewal" + return {"hostname": fqdn, "sha256": cert.fingerprint(hashes.SHA256()).hex()} + + +def reuse_query(resource_id, digest): + # Only validated identifiers enter the query. Return counts, never raw logs. + resource_id = str(UUID(resource_id)) + if not re.fullmatch("[a-f0-9]{64}", digest): + raise ValueError("Expected a SHA-256 certificate fingerprint") + return f""" +let received = AppTraces +| where Message contains 'Message received for resource_id={resource_id},' +| where isnotempty(OperationId) +| distinct OperationId; +AppTraces +| where OperationId in (received) +| summarize + reused=countif(Message contains 'Reusing certificate nexus-ssl, SHA-256 fingerprint {digest}; no ACME request required'), + issued=countif(Message contains 'Certificate issuance required:'), + completed=countif(Message contains 'Message for resource_id={resource_id},' and Message contains 'processed as True and marked complete.') +""" + + +def query_counts(response): + if response.get("error"): + raise RuntimeError("Certificate log query returned incomplete evidence") + tables = [table for table in response.get("tables", []) if table.get("name") == "PrimaryResult"] + if len(tables) != 1 or len(tables[0].get("rows", [])) != 1: + raise RuntimeError("Certificate log query did not return one evidence row") + names = [column["name"] for column in tables[0]["columns"]] + values = tables[0]["rows"][0] + if names != ["reused", "issued", "completed"] or len(values) != 3: + raise RuntimeError("Certificate log query returned unexpected columns") + if any(type(value) is not int or value < 0 for value in values): + raise RuntimeError("Certificate log query returned invalid counts") + return dict(zip(names, values, strict=True)) + + +class CertificateChecks: + def __init__(self, arm, subscription_id, tre_id): + subscription_id = str(UUID(subscription_id)) + if not re.fullmatch("[a-z0-9][a-z0-9-]{1,20}", tre_id): + raise ValueError("Expected the deployed TRE identifier") + self.arm = arm + self.tre_id = tre_id + self.group = f"/subscriptions/{subscription_id}/resourceGroups/rg-{tre_id}" + self.gateway_id = f"{self.group}/providers/Microsoft.Network/applicationGateways/agw-certs-{tre_id}" + self.ip_id = f"{self.group}/providers/Microsoft.Network/publicIPAddresses/pip-cert-nexus-{tre_id}" + storage = f"stwebcerts{tre_id}".replace("-", "") + self.storage_id = f"{self.group}/providers/Microsoft.Storage/storageAccounts/{storage}" + self.logs_id = None + + async def prepare(self): + workspace = await self.arm.request( + "GET", f"{self.group}/providers/Microsoft.OperationalInsights/workspaces/log-{self.tre_id}", LOGS_API + ) + self.logs_id = str(UUID(workspace["properties"]["customerId"])) + # Check query access before recovering or creating any shared service. + await self.query(reuse_query("00000000-0000-0000-0000-000000000000", "0" * 64)) + + async def query(self, query): + token = await asyncio.to_thread(self.arm.credential.get_token, "https://api.loganalytics.io/.default") + response = await self.arm.client.post( + f"https://api.loganalytics.azure.com/v1/workspaces/{self.logs_id}/query", + headers={"Authorization": f"Bearer {token.token}"}, + json={"query": query, "timespan": "PT6H"}, + follow_redirects=False, + ) + if response.status_code != 200: + raise RuntimeError(f"Certificate log query failed with HTTP {response.status_code}") + return query_counts(response.json()) + + async def snapshot(self, resource_path): + resource_id = str(UUID(resource_path.rsplit("/", 1)[-1])) + async with asyncio.timeout(CHECK_TIMEOUT_SECONDS): + while True: + gateway = await self.arm.request("GET", self.gateway_id, NETWORK_API) + if gateway["properties"].get("operationalState") == "Stopped": + break + await asyncio.sleep(10) + public_ip = await self.arm.request("GET", self.ip_id, NETWORK_API) + evidence = certificate_metadata(gateway, public_ip, self.group, self.tre_id, resource_id) + return {**evidence, "gateway_stopped": True} + + async def confirm_reuse(self, resource_path, digest): + query = reuse_query(resource_path.rsplit("/", 1)[-1], digest) + async with asyncio.timeout(CHECK_TIMEOUT_SECONDS): + while True: + counts = await self.query(query) + assert counts["issued"] == 0, "Reinstallation entered the ACME issuance path" + if counts["reused"] > 0 and counts["completed"] > 0: + return + await asyncio.sleep(10) + + async def confirm_removed(self): + async with asyncio.timeout(CHECK_TIMEOUT_SECONDS): + while True: + remaining = [] + for resource_id, api in ( + (self.gateway_id, NETWORK_API), + (self.ip_id, NETWORK_API), + (self.storage_id, STORAGE_API), + ): + if await self.arm.request("GET", resource_id, api, missing_ok=True) is not None: + remaining.append(resource_id) + if not remaining: + return + await asyncio.sleep(10) + + +@asynccontextmanager +async def certificate_checks(): + # Match the existing ARM helper. Do not send tokens to guessed cloud endpoints. + if get_cloud().name != "AzureCloud": + raise ValueError("Certificate lifecycle evidence currently supports AzureCloud only") + async with arm_client() as arm: + checks = CertificateChecks(arm, os.environ["ARM_SUBSCRIPTION_ID"], config.TRE_ID) + await checks.prepare() + yield checks diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index eae55f46fd..1399b034b8 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -1,5 +1,6 @@ from copy import deepcopy import asyncio +import re from contextlib import AsyncExitStack, asynccontextmanager import pytest @@ -8,6 +9,8 @@ from starlette import status from e2e_tests import config from e2e_tests.conftest import disable_and_delete_tre_resource +from e2e_tests.bundle_evidence import record_certificate_check +from e2e_tests.resources.certificate import certificate_checks from datetime import date from resources.resource import ( @@ -26,6 +29,20 @@ PROVISIONING_TIMEOUT_SECONDS = 60 * 60 CLEANUP_TIMEOUT_SECONDS = 60 * 60 CERTIFICATE_CHECK_TIMEOUT_SECONDS = 5 * 60 +CERTIFICATE_RECOVERY_TIMEOUT_SECONDS = 30 * 60 +CERTIFICATE_PROVISION_TIMEOUT_SECONDS = 45 * 60 +CERTIFICATE_LIFECYCLE_TIMEOUT_SECONDS = ( + CERTIFICATE_CHECK_TIMEOUT_SECONDS + + CERTIFICATE_RECOVERY_TIMEOUT_SECONDS + + 2 + * ( + CERTIFICATE_PROVISION_TIMEOUT_SECONDS + + 2 * CERTIFICATE_CHECK_TIMEOUT_SECONDS + + max(CLEANUP_TIMEOUT_SECONDS, FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS) + + CERTIFICATE_CHECK_TIMEOUT_SECONDS + ) + + 10 * 60 +) # Recover old resources under one shared deadline. Failed creation can spend # another cleanup deadline inside post_resource before unwinding the exit stack. SINGLE_SERVICE_TIMEOUT_SECONDS = ( @@ -230,19 +247,29 @@ async def test_create_shared_service(template_name, verify) -> None: @pytest.mark.certificate_validation -@pytest.mark.timeout(SINGLE_SERVICE_TIMEOUT_SECONDS) +@pytest.mark.timeout(CERTIFICATE_LIFECYCLE_TIMEOUT_SECONDS) async def test_create_certificate_shared_service(verify) -> None: if date.today().weekday() in [5, 6] and not config.TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS: pytest.skip("Certificate rate-limit precaution: skipping on weekends; certificate coverage is unproven.") - async with asyncio.timeout(RECOVERY_TIMEOUT_SECONDS): + async with asyncio.timeout(CERTIFICATE_CHECK_TIMEOUT_SECONDS): admin_token = await get_admin_token(verify) if await get_shared_service_by_name(strings.NEXUS_SHARED_SERVICE, verify, admin_token): pytest.fail("Nexus is present. Certificate-only validation requires an isolated environment.") - await recover_previous_shared_services(verify, strings.CERTS_SHARED_SERVICE) + template = ( + await get_resource( + f"{strings.API_SHARED_SERVICE_TEMPLATES}/{strings.CERTS_SHARED_SERVICE}", admin_token, verify + ) + )["template"] + version = template["version"] + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) or tuple(map(int, version.split("."))) < (0, 9, 0): + pytest.fail( + "Certificate reuse requires tre-shared-service-certs 0.9.0 or later. Deploy the reuse fix first." + ) payload = { "templateName": strings.CERTS_SHARED_SERVICE, + "templateVersion": version, "properties": { "display_name": f"Shared service {strings.CERTS_SHARED_SERVICE}", "description": f"{strings.CERTS_SHARED_SERVICE} deployed via e2e tests", @@ -250,26 +277,52 @@ async def test_create_certificate_shared_service(verify) -> None: "cert_name": "nexus-ssl", }, } - async with AsyncExitStack() as resources: - async with asyncio.timeout(PROVISIONING_TIMEOUT_SECONDS): - resource_path = await resources.enter_async_context(managed_shared_service(payload, verify)) - async with asyncio.timeout(CERTIFICATE_CHECK_TIMEOUT_SECONDS): - certificate = (await get_resource(f"/api{resource_path}", admin_token, verify))["sharedService"] - assert certificate["id"] == resource_path.rsplit("/", 1)[-1] - assert certificate["templateName"] == strings.CERTS_SHARED_SERVICE - assert certificate["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED - assert certificate["isEnabled"] is True - assert certificate["properties"]["domain_prefix"] == "nexus" - assert certificate["properties"]["cert_name"] == "nexus-ssl" + async with AsyncExitStack() as clients: + async with asyncio.timeout(CERTIFICATE_RECOVERY_TIMEOUT_SECONDS): + checks = await clients.enter_async_context(certificate_checks()) + await recover_previous_shared_services(verify, strings.CERTS_SHARED_SERVICE) + await checks.confirm_removed() + original = None + original_path = None + for phase in ("install", "reinstall"): + # Creation and validation deadlines expire before the exit stack + # starts cleanup. Each installation has a separate cleanup budget. + async with AsyncExitStack() as resources: + async with asyncio.timeout(CERTIFICATE_PROVISION_TIMEOUT_SECONDS): + if await get_shared_service_by_name(strings.NEXUS_SHARED_SERVICE, verify, admin_token): + pytest.fail("Nexus is present. Certificate-only validation requires an isolated environment.") + resource_path = await resources.enter_async_context(managed_shared_service(payload, verify)) + async with asyncio.timeout(2 * CERTIFICATE_CHECK_TIMEOUT_SECONDS): + certificate = (await get_resource(f"/api{resource_path}", admin_token, verify))["sharedService"] + assert certificate["id"] == resource_path.rsplit("/", 1)[-1] + assert certificate["templateName"] == strings.CERTS_SHARED_SERVICE + assert certificate["templateVersion"] == version + assert certificate["deploymentStatus"] == strings.RESOURCE_STATUS_DEPLOYED + assert certificate["isEnabled"] is True + assert certificate["properties"]["domain_prefix"] == "nexus" + assert certificate["properties"]["cert_name"] == "nexus-ssl" + evidence = await checks.snapshot(resource_path) + if original is not None: + assert resource_path != original_path, "Reinstallation must create a new service resource" + assert evidence == original, "Reinstallation did not preserve the certificate and hostname" + await checks.confirm_reuse(resource_path, original["sha256"]) + record_certificate_check(phase, resource_path, evidence, reused=original is not None) + LOGGER.info( + "Certificate %s: hostname=%s SHA-256=%s", phase, evidence["hostname"], evidence["sha256"] + ) + original = evidence + original_path = resource_path - async with asyncio.timeout(CERTIFICATE_CHECK_TIMEOUT_SECONDS): - async with AsyncClient(verify=verify) as client: - response = await client.get( - get_full_endpoint(f"/api{resource_path}"), headers=get_auth_header(admin_token), timeout=TIMEOUT - ) - assert response.status_code == status.HTTP_404_NOT_FOUND, ( - f"Certificate service remains visible after uninstall: HTTP {response.status_code}" - ) + async with asyncio.timeout(CERTIFICATE_CHECK_TIMEOUT_SECONDS): + async with AsyncClient(verify=verify) as client: + response = await client.get( + get_full_endpoint(f"/api{resource_path}"), headers=get_auth_header(admin_token), timeout=TIMEOUT + ) + assert response.status_code == status.HTTP_404_NOT_FOUND, ( + f"Certificate service remains visible after uninstall: HTTP {response.status_code}" + ) + await checks.confirm_removed() + record_certificate_check(f"{phase}-cleanup", resource_path, {}, removed=True) @pytest.mark.shared_services diff --git a/e2e_tests/unit_tests/test_certificate_evidence.py b/e2e_tests/unit_tests/test_certificate_evidence.py new file mode 100644 index 0000000000..b6ea80cb66 --- /dev/null +++ b/e2e_tests/unit_tests/test_certificate_evidence.py @@ -0,0 +1,236 @@ +"""Check public certificate evidence, correlation and bounded cleanup reads.""" + +import base64 +from copy import deepcopy +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from httpx import AsyncClient, MockTransport, Response + +from e2e_tests import bundle_evidence +from e2e_tests.resources import certificate as certs + + +SUBSCRIPTION = "11111111-1111-4111-8111-111111111111" +RESOURCE = "22222222-2222-4222-8222-222222222222" +WORKSPACE = "33333333-3333-4333-8333-333333333333" +TRE_ID = "testtre" +GROUP = f"/subscriptions/{SUBSCRIPTION}/resourceGroups/rg-{TRE_ID}" +HOSTNAME = "nexus-testtre.switzerlandnorth.cloudapp.azure.com" + + +def gateway_certificate(*, hostname=HOSTNAME, days=90, pem=False): + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(x509.NameOID.COMMON_NAME, hostname)]) + now = datetime.now(timezone.utc) + certificate = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(days=1)) + .not_valid_after(now + timedelta(days=days)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName(hostname)]), critical=False) + .sign(key, hashes.SHA256()) + ) + encoding = serialization.Encoding.PEM if pem else serialization.Encoding.DER + return base64.b64encode(certificate.public_bytes(encoding)).decode(), certificate.fingerprint(hashes.SHA256()).hex() + + +def resources(**kwargs): + data, fingerprint = gateway_certificate(**kwargs) + tags = {"tre_id": TRE_ID, "tre_shared_service_id": RESOURCE} + gateway = { + "id": f"{GROUP}/providers/Microsoft.Network/applicationGateways/agw-certs-{TRE_ID}", + "tags": tags, + "properties": { + "operationalState": "Stopped", + "sslCertificates": [ + { + "name": "cert-primary", + "properties": { + "keyVaultSecretId": f"https://kv-{TRE_ID}.vault.azure.net/secrets/nexus-ssl/abcd", + "publicCertData": data, + }, + } + ], + }, + } + ip = { + "id": f"{GROUP}/providers/Microsoft.Network/publicIPAddresses/pip-cert-nexus-{TRE_ID}", + "tags": tags, + "properties": {"dnsSettings": {"fqdn": HOSTNAME}}, + } + return gateway, ip, fingerprint + + +def table(reused=1, issued=0, completed=1): + return { + "tables": [ + { + "name": "PrimaryResult", + "columns": [{"name": n} for n in ("reused", "issued", "completed")], + "rows": [[reused, issued, completed]], + } + ] + } + + +class CertificateMetadataTests(unittest.TestCase): + def test_public_der_and_pem_produce_the_expected_sha256(self): + for pem in (False, True): + with self.subTest(pem=pem): + gateway, ip, fingerprint = resources(pem=pem) + self.assertEqual( + certs.certificate_metadata(gateway, ip, GROUP, TRE_ID, RESOURCE), + {"hostname": HOSTNAME, "sha256": fingerprint}, + ) + + def test_rejects_wrong_hostname_and_certificate_due_for_renewal(self): + for parameters in ({"hostname": "other.example.com"}, {"days": 29}): + gateway, ip, _ = resources(**parameters) + with self.subTest(parameters=parameters), self.assertRaises(AssertionError): + certs.certificate_metadata(gateway, ip, GROUP, TRE_ID, RESOURCE) + + def test_rejects_unrelated_resource_and_vault_references(self): + gateway, ip, _ = resources() + changed = deepcopy(gateway) + changed["tags"]["tre_shared_service_id"] = "another-resource" + with self.assertRaisesRegex(AssertionError, "does not belong"): + certs.certificate_metadata(changed, ip, GROUP, TRE_ID, RESOURCE) + for uri in ( + "https://other.vault.azure.net/secrets/nexus-ssl/abcd", + f"https://kv-{TRE_ID}.vault.azure.net/secrets/other/abcd", + f"https://kv-{TRE_ID}.vault.azure.net/secrets/nexus-ssl/abcd?token=secret", + ): + changed = deepcopy(gateway) + changed["properties"]["sslCertificates"][0]["properties"]["keyVaultSecretId"] = uri + with self.subTest(uri=uri), self.assertRaisesRegex(AssertionError, "core vault"): + certs.certificate_metadata(changed, ip, GROUP, TRE_ID, RESOURCE) + + def test_query_correlates_resource_trace_and_returns_only_counts(self): + query = certs.reuse_query(RESOURCE, "a" * 64) + self.assertIn(f"Message received for resource_id={RESOURCE},", query) + self.assertIn("where OperationId in (received)", query) + self.assertIn("summarize", query) + self.assertIn("no ACME request required", query) + with self.assertRaises(ValueError): + certs.reuse_query("'; AppTraces", "a" * 64) + with self.assertRaises(ValueError): + certs.reuse_query(RESOURCE, "not-a-fingerprint") + + def test_partial_missing_or_malformed_logs_are_not_evidence(self): + for response in ( + {}, + {**table(), "error": {"code": "PartialError"}}, + table(reused=True), + table(issued=-1), + {"tables": [{"name": "PrimaryResult", "rows": []}]}, + ): + with self.subTest(response=response), self.assertRaises(RuntimeError): + certs.query_counts(response) + + def test_report_drops_unapproved_metadata_and_raw_logs(self): + with tempfile.TemporaryDirectory() as folder: + report = bundle_evidence.BundleReport(Path(folder) / "report.json", {}) + with patch.object(bundle_evidence, "_ACTIVE_REPORT", report): + bundle_evidence.record_certificate_check( + "reinstall", + f"/shared-services/{RESOURCE}", + {"hostname": HOSTNAME, "sha256": "a" * 64, "password": "do-not-copy", "logs": "do-not-copy"}, + reused=True, + ) + saved = json.loads(report.filename.read_text()) + self.assertTrue(saved["certificate_checks"][0]["reuse_confirmed"]) + self.assertNotIn("do-not-copy", report.filename.read_text()) + + +class CertificateAzureReadTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.arm = SimpleNamespace(request=AsyncMock(), credential=Mock()) + self.checks = certs.CertificateChecks(self.arm, SUBSCRIPTION, TRE_ID) + + async def test_snapshot_waits_for_gateway_stop_and_reads_only_arm(self): + gateway, ip, digest = resources() + running = deepcopy(gateway) + running["properties"]["operationalState"] = "Running" + self.arm.request.side_effect = [running, gateway, ip] + with patch.object(certs.asyncio, "sleep", AsyncMock()): + result = await self.checks.snapshot(f"/shared-services/{RESOURCE}") + self.assertEqual(result, {"hostname": HOSTNAME, "sha256": digest, "gateway_stopped": True}) + self.assertTrue(all(call.args[0] == "GET" for call in self.arm.request.await_args_list)) + self.assertTrue(all("Microsoft.Network/" in call.args[1] for call in self.arm.request.await_args_list)) + + async def test_gateway_stop_has_a_finite_deadline(self): + gateway, _, _ = resources() + gateway["properties"]["operationalState"] = "Running" + self.arm.request.return_value = gateway + with patch.object(certs, "CHECK_TIMEOUT_SECONDS", 0.01), self.assertRaises(TimeoutError): + await self.checks.snapshot(f"/shared-services/{RESOURCE}") + + async def test_log_ingestion_waits_for_reuse_and_completion(self): + self.checks.query = AsyncMock(side_effect=[certs.query_counts(table(0, 0, 0)), certs.query_counts(table())]) + with patch.object(certs.asyncio, "sleep", AsyncMock()): + await self.checks.confirm_reuse(f"/shared-services/{RESOURCE}", "a" * 64) + self.assertEqual(self.checks.query.await_count, 2) + + async def test_issuance_and_missing_logs_fail_the_proof(self): + self.checks.query = AsyncMock(return_value=certs.query_counts(table(1, 1, 1))) + with self.assertRaisesRegex(AssertionError, "ACME issuance"): + await self.checks.confirm_reuse(f"/shared-services/{RESOURCE}", "a" * 64) + self.checks.query.return_value = certs.query_counts(table(0, 0, 0)) + with patch.object(certs, "CHECK_TIMEOUT_SECONDS", 0.01), self.assertRaises(TimeoutError): + await self.checks.confirm_reuse(f"/shared-services/{RESOURCE}", "a" * 64) + + async def test_cleanup_requires_gateway_ip_and_storage_absence(self): + self.arm.request.side_effect = [{"id": self.checks.gateway_id}, None, None, None, None, None] + with patch.object(certs.asyncio, "sleep", AsyncMock()): + await self.checks.confirm_removed() + self.assertEqual(self.arm.request.await_count, 6) + self.assertTrue(all(call.kwargs["missing_ok"] for call in self.arm.request.await_args_list)) + self.assertTrue(all(call.args[0] == "GET" for call in self.arm.request.await_args_list)) + + async def test_failed_cleanup_read_is_not_treated_as_absence(self): + self.arm.request.side_effect = RuntimeError("ARM GET failed with HTTP 403") + with self.assertRaisesRegex(RuntimeError, "403"): + await self.checks.confirm_removed() + + async def test_logs_query_does_not_follow_redirects_or_print_response_bodies(self): + self.checks.logs_id = WORKSPACE + self.arm.credential.get_token.return_value = SimpleNamespace(token="test-token") + code = 200 + requests = [] + + def handle(request): + requests.append(request) + if code != 200: + return Response(code, headers={"location": "https://other.example.com"}, text="private-content") + return Response(200, json=table()) + + async with AsyncClient(transport=MockTransport(handle)) as client: + self.arm.client = client + self.assertEqual(await self.checks.query("query"), {"reused": 1, "issued": 0, "completed": 1}) + for code in (302, 403, 500): + with self.subTest(code=code), self.assertRaisesRegex(RuntimeError, f"HTTP {code}") as caught: + await self.checks.query("query") + self.assertNotIn("private-content", str(caught.exception)) + self.assertEqual(len(requests), 4) + self.assertTrue(all(request.url.host == "api.loganalytics.azure.com" for request in requests)) + self.arm.credential.get_token.assert_called_with("https://api.loganalytics.io/.default") + + async def test_prepare_checks_logs_access_before_creating_resources(self): + self.arm.request.return_value = {"properties": {"customerId": WORKSPACE}} + self.checks.query = AsyncMock(return_value={"reused": 0, "issued": 0, "completed": 0}) + await self.checks.prepare() + self.assertEqual(self.checks.logs_id, WORKSPACE) + self.checks.query.assert_awaited_once() + self.assertEqual(self.arm.request.call_args.args[0], "GET") diff --git a/e2e_tests/unit_tests/test_certificate_selection.py b/e2e_tests/unit_tests/test_certificate_selection.py index 232b716c73..37c3bef92b 100644 --- a/e2e_tests/unit_tests/test_certificate_selection.py +++ b/e2e_tests/unit_tests/test_certificate_selection.py @@ -3,6 +3,8 @@ import asyncio import json from pathlib import Path +from contextlib import asynccontextmanager +from types import SimpleNamespace import unittest from unittest.mock import AsyncMock, patch @@ -27,25 +29,55 @@ def setUp(self): patch.object(shared, "disable_and_delete_shared_service_if_exists", AsyncMock()) ) self.post = self.enterContext( - patch.object(shared, "post_resource", AsyncMock(return_value=("/shared-services/certs", "certs"))) + patch.object( + shared, + "post_resource", + AsyncMock( + side_effect=[("/shared-services/certs", "certs"), ("/shared-services/reinstalled", "reinstalled")] + ), + ) ) self.cleanup = self.enterContext(patch.object(shared, "disable_and_delete_tre_resource", AsyncMock())) self.record = { "id": "certs", "templateName": shared.strings.CERTS_SHARED_SERVICE, + "templateVersion": "0.9.0", "deploymentStatus": "deployed", "isEnabled": True, "properties": {"domain_prefix": "nexus", "cert_name": "nexus-ssl"}, } - self.get = self.enterContext( - patch.object(shared, "get_resource", AsyncMock(return_value={"sharedService": self.record})) + self.template_version = "0.9.0" + + async def get_record(endpoint, *args): + return {"sharedService": {**self.record, "id": endpoint.rsplit("/", 1)[-1]}} + + self.get = AsyncMock(side_effect=get_record) + + async def get_resource(endpoint, *args): + if endpoint.startswith(shared.strings.API_SHARED_SERVICE_TEMPLATES): + return {"template": {"version": self.template_version}} + return await self.get(endpoint, *args) + + self.enterContext(patch.object(shared, "get_resource", get_resource)) + self.checks = SimpleNamespace( + snapshot=AsyncMock( + return_value={"hostname": "nexus-tre.test", "sha256": "a" * 64, "gateway_stopped": True} + ), + confirm_reuse=AsyncMock(), + confirm_removed=AsyncMock(), ) + + @asynccontextmanager + async def checks(): + yield self.checks + + self.enterContext(patch.object(shared, "certificate_checks", checks)) self.lookup_status = 404 self.lookups = [] def lookup(request): self.assertEqual(request.method, "GET") - self.assertEqual(request.url.path, "/api/shared-services/certs") + self.assertIn(request.url.path, ("/api/shared-services/certs", "/api/shared-services/reinstalled")) self.assertEqual(request.headers["Authorization"], "Bearer token") self.lookups.append(request) return httpx.Response(self.lookup_status, request=request) @@ -59,15 +91,18 @@ def lookup(request): async def test_creates_only_certificates_without_nexus_consent_and_confirms_removal(self): await shared.test_create_certificate_shared_service(True) - self.post.assert_awaited_once() + self.assertEqual(self.post.await_count, 2) arguments = self.post.await_args.kwargs self.assertEqual(arguments["payload"]["templateName"], shared.strings.CERTS_SHARED_SERVICE) self.assertTrue(arguments["cleanup_failed_create"]) schema = Path(__file__).resolve().parents[2] / "templates/shared_services/certs/template_schema.json" validate(arguments["payload"]["properties"], json.loads(schema.read_text())) self.recover.assert_awaited_once_with(shared.strings.CERTS_SHARED_SERVICE, True) - self.cleanup.assert_awaited_once_with("/shared-services/certs", True) - self.assertEqual(len(self.lookups), 1) + self.assertEqual(self.cleanup.await_count, 2) + self.assertEqual(len(self.lookups), 2) + self.checks.confirm_removed.assert_awaited() + self.assertEqual(self.checks.confirm_removed.await_count, 3) + self.checks.confirm_reuse.assert_awaited_once_with("/shared-services/reinstalled", "a" * 64) async def test_existing_nexus_prevents_recovery_and_creation(self): self.find_service.return_value = {"id": "existing-nexus"} @@ -91,8 +126,8 @@ async def test_explicit_weekend_override_allows_the_certificate_case(self): self.date.today.return_value.weekday.return_value = 5 with patch.object(shared.config, "TEST_RUN_CERTIFICATE_TESTS_ON_WEEKENDS", True): await shared.test_create_certificate_shared_service(True) - self.post.assert_awaited_once() - self.cleanup.assert_awaited_once() + self.assertEqual(self.post.await_count, 2) + self.assertEqual(self.cleanup.await_count, 2) async def test_wrong_deployed_record_fails_and_still_cleans_up(self): self.record["deploymentStatus"] = "deployment_failed" @@ -110,7 +145,7 @@ async def test_body_error_survives_a_cleanup_failure(self): self.assertIn("Delete failed", failure.__notes__[0]) async def test_nexus_appearing_during_the_case_retains_the_certificate(self): - self.find_service.side_effect = [None, {"id": "new-nexus"}] + self.find_service.side_effect = [None, None, {"id": "new-nexus"}] with self.assertRaisesRegex(RuntimeError, "Retaining certificate dependency"): await shared.test_create_certificate_shared_service(True) self.cleanup.assert_not_awaited() @@ -119,6 +154,7 @@ async def test_nexus_appearing_during_the_case_retains_the_certificate(self): async def test_visible_record_after_deletion_fails_the_case(self): for code in (200, 403, 500): with self.subTest(code=code): + self.post.side_effect = [("/shared-services/certs", "certs")] self.lookup_status = code with self.assertRaisesRegex(AssertionError, f"HTTP {code}"): await shared.test_create_certificate_shared_service(True) @@ -161,3 +197,59 @@ async def recover(*args): with self.assertRaises(TimeoutError): await shared.test_create_certificate_shared_service(True) self.post.assert_not_awaited() + + async def test_old_bundle_fails_before_recovery_or_new_creation(self): + for version in ("0.7.13", "0.8.9", "0.9.0-rc1", "invalid"): + with self.subTest(version=version): + self.template_version = version + with self.assertRaisesRegex(pytest.fail.Exception, "Deploy the reuse fix first"): + await shared.test_create_certificate_shared_service(True) + self.recover.assert_not_awaited() + self.post.assert_not_awaited() + + async def test_changed_thumbprint_fails_and_cleans_both_installations(self): + original = self.checks.snapshot.return_value + self.checks.snapshot.side_effect = [original, {**original, "sha256": "b" * 64}] + with self.assertRaisesRegex(AssertionError, "did not preserve"): + await shared.test_create_certificate_shared_service(True) + self.assertEqual(self.cleanup.await_count, 2) + self.checks.confirm_reuse.assert_not_awaited() + + async def test_missing_reuse_log_fails_and_cleans_reinstalled_service(self): + self.checks.confirm_reuse.side_effect = TimeoutError("No reuse evidence") + with self.assertRaisesRegex(TimeoutError, "No reuse evidence"): + await shared.test_create_certificate_shared_service(True) + self.assertEqual(self.cleanup.await_count, 2) + + async def test_first_cleanup_failure_prevents_reinstallation(self): + self.cleanup.side_effect = RuntimeError("Delete failed") + with self.assertRaisesRegex(RuntimeError, "Delete failed"): + await shared.test_create_certificate_shared_service(True) + self.post.assert_awaited_once() + + async def test_arm_cleanup_failure_prevents_reinstallation(self): + self.checks.confirm_removed.side_effect = [None, TimeoutError("Gateway remains")] + with self.assertRaisesRegex(TimeoutError, "Gateway remains"): + await shared.test_create_certificate_shared_service(True) + self.post.assert_awaited_once() + + async def test_nexus_appearing_before_reinstall_prevents_new_creation(self): + self.find_service.side_effect = [None, None, None, {"id": "new-nexus"}] + with self.assertRaisesRegex(pytest.fail.Exception, "Nexus is present"): + await shared.test_create_certificate_shared_service(True) + self.post.assert_awaited_once() + self.cleanup.assert_awaited_once() + + def test_lifecycle_timeout_leaves_job_setup_and_report_time(self): + self.assertLess(shared.CERTIFICATE_LIFECYCLE_TIMEOUT_SECONDS, 290 * 60) + required = ( + shared.CERTIFICATE_CHECK_TIMEOUT_SECONDS + + shared.CERTIFICATE_RECOVERY_TIMEOUT_SECONDS + + 2 + * ( + shared.CERTIFICATE_PROVISION_TIMEOUT_SECONDS + + 3 * shared.CERTIFICATE_CHECK_TIMEOUT_SECONDS + + max(shared.CLEANUP_TIMEOUT_SECONDS, shared.FAILED_CREATE_CLEANUP_TIMEOUT_SECONDS) + ) + ) + self.assertGreater(shared.CERTIFICATE_LIFECYCLE_TIMEOUT_SECONDS, required) From f46d3071c8e34e4c3e537aa035231ffc19877d68 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 14 Oct 2026 14:51:18 +0100 Subject: [PATCH 11/17] Clarify E2E helper documentation (#5119) --- docs/tre-developers/end-to-end-tests.md | 12 ++---------- e2e_tests/unit_tests/README.md | 19 ++++++------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 7b9fd324c1..62195a5a14 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -25,16 +25,8 @@ Use the "Run and Debug" panel within Visual Studio Code, select "E2E Extended", - This will also use `/workspaces/AzureTRE/core/private.env` file for other values. -## Investigate operation-polling failures - -Unexpected HTTP responses fail operation polling. HTTP 500 is not retried or treated as successful deletion. -The error log records a UTC timestamp, method, recognised operation path, status, media type, body size and coarse body classification. -It includes allowlisted correlation headers only when their values match supported identifier formats. -Unrecognised paths and header values are omitted or redacted. The diagnostic record excludes response bodies, credentials, host names and URL parameters. - -If a poll fails, use its timestamp, operation ID and available correlation IDs to find the API exception and App Service request record. -Check API exception telemetry before a validation run. Record terminal resource deletion separately from the failed poll. -These diagnostics help investigate the failure. They do not establish its cause or guarantee that server telemetry is available. +If an E2E test fails while waiting for an operation, check its test log. +Use the timestamp and operation ID to find the matching API logs. ## Airlock draft deletion diff --git a/e2e_tests/unit_tests/README.md b/e2e_tests/unit_tests/README.md index 14fb1700d6..9d434e576c 100644 --- a/e2e_tests/unit_tests/README.md +++ b/e2e_tests/unit_tests/README.md @@ -7,18 +7,11 @@ python -m pip install -r e2e_tests/unit_tests/requirements.txt PYTHONPATH=.:e2e_tests python -m unittest discover -s e2e_tests/unit_tests -v ``` -The tests use an in-memory HTTP transport and simulated token expiry. They need -no Azure credentials and do not create resources or wait for real token expiry. -They exercise deployment, update and teardown polling, separate Airlock polling -credentials, and authentication failures without replaying resource mutations. +These unit tests check the code used by the E2E test suite. +They cover authentication, operation polling, cleanup, timeouts, test selection and result reporting. -The E2E helpers retain the Azure Identity credential and scope. Each request -asks that credential for a token so the SDK can use its cache or renew an expiring -token. Literal token strings remain supported but cannot be renewed. +They run without Azure credentials or deployed resources. +Cases named after SQL, Nexus or certificates check the E2E setup, validation and cleanup for those cases. -These tests check the request and renewal behaviour locally. A live extended -test run is still needed to validate the fix against Azure. - -The bundle-selector tests also collect the real Azure SQL case without running -fixtures. Small isolated pytest runs check exact parameter selection, skipped -cases, teardown failures and artefact redaction. No Azure deployment is required. +The tests stay beside the E2E helpers they exercise. +Live E2E tests are still required to validate deployment behaviour against Azure. From 2040c9838c8fa895ff2907f785e2338861d74f95 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 14 Oct 2026 15:08:52 +0100 Subject: [PATCH 12/17] Keep E2E polling logs free of response content (#5119) Address review feedback on #5255. Keep recognised operation paths and states in progress, cancellation and failure logs. Exclude response messages, step output and URL parameters. Add regression coverage and use realistic operation IDs in cleanup fixtures. --- CHANGELOG.md | 1 + e2e_tests/resources/deployment.py | 38 ++++++++++- e2e_tests/resources/resource.py | 29 ++++---- e2e_tests/unit_tests/test_disable_deadline.py | 12 +++- e2e_tests/unit_tests/test_fixture_cleanup.py | 8 ++- .../unit_tests/test_polling_diagnostics.py | 68 ++++++++++++++++++- 6 files changed, 133 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef13f3edee..ee20736dcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Keep response content and URL parameters out of E2E operation polling logs, including cancellation and deployment failures. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Protect active validation environments, require successful smoke tests before custom tests, and give E2E cleanup separate deadlines. Forward explicit Nexus consent and verify TLS for identity authentication. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) * Validate Airlock sort fields against the supported options and avoid returning database error details to callers. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/e2e_tests/resources/deployment.py b/e2e_tests/resources/deployment.py index 6189d6374e..b1969b0f75 100644 --- a/e2e_tests/resources/deployment.py +++ b/e2e_tests/resources/deployment.py @@ -12,6 +12,28 @@ rf"/api/(?:workspaces/{_UUID}/(?:workspace-services/{_UUID}/(?:user-resources/{_UUID}/)?)?" rf"|shared-services/{_UUID}/)?operations/{_UUID}" ) +_OPERATION_STATES = { + strings.RESOURCE_STATUS_AWAITING_DEPLOYMENT, + strings.RESOURCE_STATUS_DEPLOYING, + strings.RESOURCE_STATUS_DEPLOYED, + strings.RESOURCE_STATUS_DEPLOYMENT_FAILED, + strings.RESOURCE_STATUS_AWAITING_DELETION, + strings.RESOURCE_STATUS_DELETING, + strings.RESOURCE_STATUS_DELETED, + strings.RESOURCE_STATUS_DELETING_FAILED, + strings.RESOURCE_STATUS_AWAITING_UPDATE, + strings.RESOURCE_STATUS_UPDATING, + strings.RESOURCE_STATUS_UPDATED, + strings.RESOURCE_STATUS_UPDATING_FAILED, + strings.RESOURCE_STATUS_AWAITING_ACTION, + strings.RESOURCE_ACTION_STATUS_INVOKING, + strings.RESOURCE_ACTION_STATUS_SUCCEEDED, + strings.RESOURCE_ACTION_STATUS_FAILED, + strings.RESOURCE_ACTION_STATUS_PIPELINE_RUNNING, + strings.RESOURCE_ACTION_STATUS_PIPELINE_FAILED, + strings.RESOURCE_ACTION_STATUS_PIPELINE_SUCCEEDED, + "not yet observed", +} _CORRELATION_HEADERS = { "x-ms-request-id": _UUID, "x-ms-correlation-request-id": _UUID, @@ -21,9 +43,19 @@ } +def operation_log_path(operation_endpoint): + """Keep recognised operation paths without URL parameters or fragments.""" + path = operation_endpoint.split("?", 1)[0].split("#", 1)[0] + return path if _OPERATION_PATH.fullmatch(path) else "" + + +def operation_log_state(state): + """Exclude arbitrary response content from operation-state diagnostics.""" + return state if isinstance(state, str) and state in _OPERATION_STATES else "" + + def polling_failure_details(response, operation_endpoint): """Return bounded metadata. Never copy bodies, URL parameters or arbitrary headers.""" - path = operation_endpoint.split("?", 1)[0].split("#", 1)[0] media_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower() allowed_media_types = {"application/json", "application/problem+json", "text/plain", "text/html"} prefix = response.content[:64].lstrip().lower() @@ -38,7 +70,7 @@ def polling_failure_details(response, operation_endpoint): return { "utc_time": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), "method": "GET", - "path": path if _OPERATION_PATH.fullmatch(path) else "", + "path": operation_log_path(operation_endpoint), "status": response.status_code, "content_type": media_type if media_type in allowed_media_types else "other_or_missing", "body_bytes": len(response.content), @@ -99,7 +131,7 @@ async def check_deployment(client, operation_endpoint, access_token): elif response.status_code == 404: # 404 indicates the resource has been deleted (filtered out by get_by_id methods) # This is a valid state for deletion operations - LOGGER.info(f"Resource not found (404) - treating as deleted: {operation_endpoint}") + LOGGER.info("Resource not found (404) - treating as deleted: %s", operation_log_path(operation_endpoint)) return strings.RESOURCE_STATUS_DELETED, "Resource has been deleted", "" else: LOGGER.error("Operation polling failed: %s", polling_failure_details(response, operation_endpoint)) diff --git a/e2e_tests/resources/resource.py b/e2e_tests/resources/resource.py index aa5fb7d91a..242bfd24e1 100644 --- a/e2e_tests/resources/resource.py +++ b/e2e_tests/resources/resource.py @@ -6,7 +6,13 @@ from httpx import AsyncClient, Timeout from starlette import status from e2e_tests.helpers import assert_status, get_auth_header, get_full_endpoint -from e2e_tests.resources.deployment import delete_done, install_done, patch_done +from e2e_tests.resources.deployment import ( + delete_done, + install_done, + operation_log_path, + operation_log_state, + patch_done, +) from resources import strings @@ -163,30 +169,29 @@ async def temporary_resource(payload, endpoint, access_token, verify): async def wait_for(func, client, operation_endpoint, access_token, failure_states: list): - done_state, message, operation_steps = "not yet observed", "", "" - LOGGER.info(f"WAITING FOR OP: {operation_endpoint}") + done_state = "not yet observed" + log_path = operation_log_path(operation_endpoint) + LOGGER.info("WAITING FOR OP: %s", log_path) try: - done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) + done, done_state, _, _ = await func(client, operation_endpoint, access_token) record_operation(operation_endpoint, done_state, done) while not done: await asyncio.sleep(30) - done, done_state, message, operation_steps = await func(client, operation_endpoint, access_token) + done, done_state, _, _ = await func(client, operation_endpoint, access_token) record_operation(operation_endpoint, done_state, done) - LOGGER.info(f"{done}, {done_state}, {message}") + LOGGER.info("Operation %s. Last observed state: %s", log_path, operation_log_state(done_state)) except asyncio.CancelledError: LOGGER.error( - "Stopped waiting for operation %s. Last observed state: %s; message: %s; steps: %s", - operation_endpoint, - done_state, - message, - operation_steps, + "Stopped waiting for operation %s. Last observed state: %s", + log_path, + operation_log_state(done_state), ) raise try: assert done_state not in failure_states except Exception: - LOGGER.exception(f"Failed to deploy. Status message: {message}.\n{operation_steps}") + LOGGER.exception("Operation %s failed. Last observed state: %s", log_path, operation_log_state(done_state)) raise return done_state diff --git a/e2e_tests/unit_tests/test_disable_deadline.py b/e2e_tests/unit_tests/test_disable_deadline.py index a008925182..b6dcc5a0c5 100644 --- a/e2e_tests/unit_tests/test_disable_deadline.py +++ b/e2e_tests/unit_tests/test_disable_deadline.py @@ -8,6 +8,11 @@ from e2e_tests import conftest as fixtures, helpers from e2e_tests.resources import resource +OPERATIONS = { + "disable": "2913b8be-2594-4088-8d93-42a296fe69cc", + "delete": "05c22c61-9d22-49d7-b333-65b86a0d70dd", +} + class DisableDeadlineTests(unittest.IsolatedAsyncioTestCase): async def run_cleanup(self, stall): @@ -20,13 +25,14 @@ async def handle(request): elif request.method == "DELETE": phase, state, code = "delete", "deleting", 200 else: - phase, code = request.url.path.rsplit("/", 1)[-1], 200 + operation_id = request.url.path.rsplit("/", 1)[-1] + phase, code = {value: key for key, value in OPERATIONS.items()}[operation_id], 200 if phase == "disable": await asyncio.sleep(1 if stall else 0.001) state = {"disable": "updated", "delete": "deleted"}[phase] return Response( code, - headers={"Location": f"/api/operations/{phase}"}, + headers={"Location": f"/api/operations/{OPERATIONS[phase]}"}, json={ "operation": { "resourceId": "owned-workspace", @@ -55,6 +61,6 @@ async def test_stalled_disable_has_bounded_failure_and_does_not_send_conflicting with self.assertRaisesRegex(TimeoutError, "Cleanup of /workspaces/owned-workspace exceeded"): await asyncio.wait_for(self.run_cleanup(True), 0.5) self.assertEqual([method for method, _ in self.requests], ["PATCH", "GET"]) - self.assertIn("/api/operations/disable", "\n".join(logs.output)) + self.assertIn(f"/api/operations/{OPERATIONS['disable']}", "\n".join(logs.output)) await asyncio.sleep(0.01) self.assertEqual([method for method, _ in self.requests], ["PATCH", "GET"]) diff --git a/e2e_tests/unit_tests/test_fixture_cleanup.py b/e2e_tests/unit_tests/test_fixture_cleanup.py index 6669fdb678..661b7184bd 100644 --- a/e2e_tests/unit_tests/test_fixture_cleanup.py +++ b/e2e_tests/unit_tests/test_fixture_cleanup.py @@ -422,11 +422,13 @@ def unmanaged_fixture(): class CleanupPollingDiagnosticsTests(unittest.IsolatedAsyncioTestCase): async def test_cancellation_records_last_observed_operation_state(self): + endpoint = "/api/operations/2913b8be-2594-4088-8d93-42a296fe69cc" check = AsyncMock(return_value=(False, "pipeline_running", "Waiting for resource", "Step 1")) with self.assertLogs(resource.LOGGER, level="ERROR") as logs: with self.assertRaises(TimeoutError): async with asyncio.timeout(0.01): - await resource.wait_for(check, None, "/api/operations/cleanup", "token", []) - self.assertIn("/api/operations/cleanup", logs.output[0]) + await resource.wait_for(check, None, endpoint, "token", []) + self.assertIn(endpoint, logs.output[0]) self.assertIn("pipeline_running", logs.output[0]) - self.assertIn("Waiting for resource", logs.output[0]) + self.assertNotIn("Waiting for resource", logs.output[0]) + self.assertNotIn("Step 1", logs.output[0]) diff --git a/e2e_tests/unit_tests/test_polling_diagnostics.py b/e2e_tests/unit_tests/test_polling_diagnostics.py index 8451030d08..caa6a23b6f 100644 --- a/e2e_tests/unit_tests/test_polling_diagnostics.py +++ b/e2e_tests/unit_tests/test_polling_diagnostics.py @@ -1,12 +1,13 @@ """Keep useful HTTP failure evidence without exposing response data or credentials.""" +import asyncio import unittest -from unittest.mock import patch +from unittest.mock import AsyncMock, patch from httpx import AsyncClient, MockTransport, Response from e2e_tests import helpers -from e2e_tests.resources import deployment, strings +from e2e_tests.resources import deployment, resource, strings WORKSPACE = "9f293650-29c7-4291-a074-c5c0aaedd7b7" @@ -130,3 +131,66 @@ async def test_success_and_deletion_keep_the_existing_results(self): state, _, _ = await deployment.check_deployment(client, ENDPOINT, "token") self.assertEqual(state, expected) log.assert_not_called() + + +class ResourcePollingDiagnosticsTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.enterContext(patch.object(resource, "record_operation")) + + async def test_cancellation_excludes_response_content_and_url_parameters(self): + check = AsyncMock(return_value=(False, "pipeline_running", "message-secret", "step-secret")) + with patch.object(resource.asyncio, "sleep", new=AsyncMock(side_effect=asyncio.CancelledError)): + with self.assertLogs(resource.LOGGER, level="INFO") as logs: + with self.assertRaises(asyncio.CancelledError): + await resource.wait_for( + check, None, ENDPOINT + "?sig=query-secret#fragment-secret", "token-secret", [] + ) + log = "\n".join(logs.output) + self.assertIn(ENDPOINT, log) + self.assertIn("pipeline_running", log) + for secret in ("message-secret", "step-secret", "query-secret", "fragment-secret", "token-secret"): + self.assertNotIn(secret, log) + + async def test_progress_and_terminal_logs_exclude_response_content(self): + for state in ("deployed", "deployment_failed"): + with self.subTest(state=state): + check = AsyncMock( + side_effect=[ + (False, "pipeline_running", "message-secret", "step-secret"), + (True, state, "message-secret", "step-secret"), + ] + ) + with patch.object(resource.asyncio, "sleep", new=AsyncMock()): + with self.assertLogs(resource.LOGGER, level="INFO") as logs: + if state == "deployment_failed": + with self.assertRaises(AssertionError): + await resource.wait_for(check, None, ENDPOINT, "token-secret", [state]) + else: + result = await resource.wait_for(check, None, ENDPOINT, "token-secret", []) + self.assertEqual(result, state) + log = "\n".join(logs.output) + self.assertIn(ENDPOINT, log) + self.assertIn(state, log) + for secret in ("message-secret", "step-secret", "token-secret"): + self.assertNotIn(secret, log) + + async def test_cancellation_before_first_response_preserves_unknown_state(self): + check = AsyncMock(side_effect=asyncio.CancelledError) + with self.assertLogs(resource.LOGGER, level="ERROR") as logs: + with self.assertRaises(asyncio.CancelledError): + await resource.wait_for(check, None, ENDPOINT, "token-secret", []) + self.assertIn(ENDPOINT, logs.output[0]) + self.assertIn("not yet observed", logs.output[0]) + self.assertNotIn("token-secret", logs.output[0]) + + async def test_unknown_paths_and_states_are_redacted_and_bounded(self): + secret = "response-secret" * 1000 + check = AsyncMock(return_value=(False, secret, secret, secret)) + with patch.object(resource.asyncio, "sleep", new=AsyncMock(side_effect=asyncio.CancelledError)): + with self.assertLogs(resource.LOGGER, level="INFO") as logs: + with self.assertRaises(asyncio.CancelledError): + await resource.wait_for(check, None, "/api/operations/" + secret, "token-secret", []) + log = "\n".join(logs.output) + self.assertIn("", log) + self.assertNotIn("response-secret", log) + self.assertLess(len(log), 500) From 9ad8d125c101736be193edac6ced6cb5125d06b6 Mon Sep 17 00:00:00 2001 From: Jay W Date: Wed, 14 Oct 2026 15:09:23 +0100 Subject: [PATCH 13/17] Validate shared-service command arguments before dispatch (#5119) Address review feedback on #5255. Accept at most one hexadecimal SHA alongside explicit Nexus consent. Reject malformed arguments for internal and external PRs before dispatch. Add parser regressions and bump build-scripts to 1.1.4. --- .github/scripts/build.js | 7 +++++++ .github/scripts/build.test.js | 34 +++++++++++++++++++++++++++++++ .github/scripts/package-lock.json | 4 ++-- .github/scripts/package.json | 2 +- CHANGELOG.md | 1 + 5 files changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/scripts/build.js b/.github/scripts/build.js index 8c00b25df9..7b23ccb8e9 100644 --- a/.github/scripts/build.js +++ b/.github/scripts/build.js @@ -131,6 +131,13 @@ async function getCommandFromComment({ core, context, github }) { break; } + const shaParts = parts.slice(1).filter(part => part !== ACCEPT_NEXUS_EULA_FLAG); + if (shaParts.length > 1 || shaParts.some(part => !/^[0-9a-f]{7,40}$/i.test(part))) { + const message = `:warning: Invalid shared-service arguments. No deployment or tests were started. Use \`${commandText} [] ${ACCEPT_NEXUS_EULA_FLAG}\` with one optional SHA and one exact consent flag.`; + await addActionComment({ github }, repoOwner, repoName, prNumber, commentUsername, commentLink, message); + break; + } + const runTests = await handleTestCommand({ core, github }, parts, "shared service tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink }); if (runTests) { command = "run-tests-shared-services"; diff --git a/.github/scripts/build.test.js b/.github/scripts/build.test.js index fae4324960..2f4f7297ae 100644 --- a/.github/scripts/build.test.js +++ b/.github/scripts/build.test.js @@ -498,6 +498,40 @@ describe('getCommandFromComment', () => { }); }); + test.each([ + '/test-shared-services 2345678 accept_nexus_eula', + '/test-shared-services accept_nexus_eula 2345678', + '/test-shared-services ' + 'a'.repeat(40) + ' accept_nexus_eula', + ])('accepts one optional SHA on an internal PR: %s', async body => { + const context = createCommentContext({ body }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-shared-services'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('true'); + }); + + describe.each([ + { description: 'internal', pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES, authorUsername: 'admin' }, + { description: 'external', pullRequestNumber: PR_NUMBER.FORK_NON_DOCS_CHANGES, authorUsername: 'non-contributor' }, + ])('argument validation for $description PRs', ({ pullRequestNumber, authorUsername }) => { + test.each([ + '/test-shared-services typo accept_nexus_eula', + '/test-shared-services accept_nexus_eula typo', + '/test-shared-services 234567 accept_nexus_eula', + '/test-shared-services ' + 'a'.repeat(41) + ' accept_nexus_eula', + '/test-shared-services 2345678 2345678 accept_nexus_eula', + '/test-shared-services 2345678 accept_nexus_eula unexpected', + ])('does not dispatch malformed arguments: %s', async body => { + const context = createCommentContext({ body, pullRequestNumber, authorUsername }); + await getCommandFromComment({ core, context, github }); + expect(outputFor(mockCoreSetOutput, 'command')).toBe('none'); + expect(outputFor(mockCoreSetOutput, 'acceptNexusEula')).toBe('false'); + expect(mockGithubRestIssuesCreateComment).toHaveComment({ + owner: 'someOwner', repo: 'someRepo', issue_number: pullRequestNumber, + bodyMatcher: /Invalid shared-service arguments.*No deployment or tests were started/, + }); + }); + }); + test.each([ '/test-shared-services', '/test-shared-services accept_nexus_eula=false', diff --git a/.github/scripts/package-lock.json b/.github/scripts/package-lock.json index d34eeb9acb..d4495bbc6d 100644 --- a/.github/scripts/package-lock.json +++ b/.github/scripts/package-lock.json @@ -1,12 +1,12 @@ { "name": "build-scripts", - "version": "1.1.3", + "version": "1.1.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "build-scripts", - "version": "1.1.3", + "version": "1.1.4", "license": "MIT", "devDependencies": { "jest": "^30.2.0", diff --git a/.github/scripts/package.json b/.github/scripts/package.json index 06cf6690be..3aa4300dfa 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -1,6 +1,6 @@ { "name": "build-scripts", - "version": "1.1.3", + "version": "1.1.4", "description": "Scripts used in the GitHub workflows", "main": "build.js", "license": "MIT", diff --git a/CHANGELOG.md b/CHANGELOG.md index ee20736dcf..852e668d5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Reject invalid or duplicate SHA arguments before starting shared-service tests. (`build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Keep response content and URL parameters out of E2E operation polling logs, including cancellation and deployment failures. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Protect active validation environments, require successful smoke tests before custom tests, and give E2E cleanup separate deadlines. Forward explicit Nexus consent and verify TLS for identity authentication. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) From 7d63e09a6e3c10bbee8fd8c1ea1e78cbd49cf3ae Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 15 Oct 2026 16:09:56 +0100 Subject: [PATCH 14/17] Fix certificate lifecycle template lookup (#5158) Read the shared-service template response directly and correct the unit-test fixture to match the API response. Observed in certificate validation run 38755824404, attempt 1: https://github.com/microsoft/AzureTRE/actions/runs/38755824404/job/116495438575 Prepared for PR #5255. --- CHANGELOG.md | 1 + e2e_tests/test_shared_services.py | 8 +++----- e2e_tests/unit_tests/test_certificate_selection.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 852e668d5e..c676499247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo BUG FIXES: * Reject invalid or duplicate SHA arguments before starting shared-service tests. (`build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) +* Read the shared-service template response correctly before certificate lifecycle validation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Keep response content and URL parameters out of E2E operation polling logs, including cancellation and deployment failures. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Protect active validation environments, require successful smoke tests before custom tests, and give E2E cleanup separate deadlines. Forward explicit Nexus consent and verify TLS for identity authentication. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Restrict Airlock submission, cancellation and draft file links to the request creator or a workspace owner, and align the advertised actions with these permissions. (`API` 0.28.3) ([#5251](https://github.com/microsoft/AzureTRE/pull/5251)) diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index 1399b034b8..c38fd4dc82 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -256,11 +256,9 @@ async def test_create_certificate_shared_service(verify) -> None: admin_token = await get_admin_token(verify) if await get_shared_service_by_name(strings.NEXUS_SHARED_SERVICE, verify, admin_token): pytest.fail("Nexus is present. Certificate-only validation requires an isolated environment.") - template = ( - await get_resource( - f"{strings.API_SHARED_SERVICE_TEMPLATES}/{strings.CERTS_SHARED_SERVICE}", admin_token, verify - ) - )["template"] + template = await get_resource( + f"{strings.API_SHARED_SERVICE_TEMPLATES}/{strings.CERTS_SHARED_SERVICE}", admin_token, verify + ) version = template["version"] if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) or tuple(map(int, version.split("."))) < (0, 9, 0): pytest.fail( diff --git a/e2e_tests/unit_tests/test_certificate_selection.py b/e2e_tests/unit_tests/test_certificate_selection.py index 37c3bef92b..65593c2f1c 100644 --- a/e2e_tests/unit_tests/test_certificate_selection.py +++ b/e2e_tests/unit_tests/test_certificate_selection.py @@ -55,7 +55,7 @@ async def get_record(endpoint, *args): async def get_resource(endpoint, *args): if endpoint.startswith(shared.strings.API_SHARED_SERVICE_TEMPLATES): - return {"template": {"version": self.template_version}} + return {"name": shared.strings.CERTS_SHARED_SERVICE, "version": self.template_version} return await self.get(endpoint, *args) self.enterContext(patch.object(shared, "get_resource", get_resource)) From 4f99399e301701c343d86c6c2766b1f2eae48bf0 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 15 Oct 2026 17:03:26 +0100 Subject: [PATCH 15/17] Read public certificate evidence through the resource processor (#5158) Handle gateway responses that omit publicCertData. Read only the referenced public certificate through a temporary managed Run Command, retain certificate validation, and remove the command after use. Scope reuse evidence to installation message spans. Observed in run 38758184044, job 116503447099: https://github.com/microsoft/AzureTRE/actions/runs/38758184044/job/116503447099 Prepared for PR #5255. Full live lifecycle validation is pending. --- CHANGELOG.md | 1 + docs/tre-developers/end-to-end-tests.md | 10 +- e2e_tests/bundle_coverage.json | 2 +- e2e_tests/resources/certificate.py | 110 +++++++++++- e2e_tests/resources/certificate_probe.py | 68 ++++++++ .../unit_tests/test_certificate_evidence.py | 56 +++++- .../unit_tests/test_certificate_probe.py | 163 ++++++++++++++++++ 7 files changed, 396 insertions(+), 14 deletions(-) create mode 100644 e2e_tests/resources/certificate_probe.py create mode 100644 e2e_tests/unit_tests/test_certificate_probe.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c676499247..2496fe247f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Read missing gateway certificate metadata through the resource processor and isolate reuse evidence by installation message span. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Reject invalid or duplicate SHA arguments before starting shared-service tests. (`build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Read the shared-service template response correctly before certificate lifecycle validation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Keep response content and URL parameters out of E2E operation polling logs, including cancellation and deployment failures. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) diff --git a/docs/tre-developers/end-to-end-tests.md b/docs/tre-developers/end-to-end-tests.md index 62195a5a14..32d3da41e3 100644 --- a/docs/tre-developers/end-to-end-tests.md +++ b/docs/tre-developers/end-to-end-tests.md @@ -83,10 +83,14 @@ Reinstallation must preserve the SHA-256 certificate fingerprint and produce a m After each uninstall, the case requires API HTTP 404 and ARM confirmation that the gateway, public IP and challenge storage are absent. The bundle report records public metadata, reuse and cleanup results. It does not require Nexus EULA consent. -The evidence helper currently supports `AzureCloud`. It uses the existing deployment identity for ARM and Log Analytics reads. -It checks log access before resource changes. It does not read private keys, open vault network access or add permissions. +The evidence helper currently supports `AzureCloud`. It uses the existing deployment identity for ARM and Log Analytics access. +If the gateway omits public certificate data, the helper reads the exact referenced certificate version through the resource-processor VM. +This uses a temporary managed Run Command and the VM's existing identity and private network access. +The deployment identity must be able to create, read and delete that command on the resource-processor VM. +The helper removes the command after success, failure or cancellation. It reads public certificate data only, without changing vault access or permissions. +Log evidence is limited to the installation's message spans, so other installations in the same trace cannot affect the result. Only aggregate log counts enter the test process. Missing, partial or delayed evidence beyond the deadline fails the case. -See [Application Gateway public certificate metadata](https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.management.network.models.applicationgatewaysslcertificate.publiccertdata) +See [Key Vault public certificate retrieval](https://learn.microsoft.com/en-us/rest/api/keyvault/certificates/get-certificate/get-certificate) and [Azure Monitor query authentication](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/api/access-api). Each installation has a 45-minute provisioning deadline. Cleanup has its own 60-minute deadline. diff --git a/e2e_tests/bundle_coverage.json b/e2e_tests/bundle_coverage.json index 23c07e3fff..d443823a66 100644 --- a/e2e_tests/bundle_coverage.json +++ b/e2e_tests/bundle_coverage.json @@ -62,7 +62,7 @@ ], "limitations": [ "Requires tre-shared-service-certs 0.9.0 or later, the deployed firewall and an isolated AzureCloud environment. Refuses to start while Nexus exists. Only earlier E2E certificate resources can be recovered.", - "Uses the existing deployment identity for ARM and Log Analytics reads. Requires AppTraces and read access. Does not read Key Vault secrets or change vault network rules or permissions.", + "Uses the existing deployment identity for ARM and Log Analytics access. If the gateway omits public certificate data, reads the exact version through a temporary managed Run Command on the resource-processor VM. Requires command create/read/delete access. Does not read Key Vault secrets or change vault network rules or permissions.", "Compares the public gateway certificate fingerprint and hostname across uninstall and reinstall. Requires a matching resource processor reuse log and rejects issuance evidence. Missing or partial log evidence fails the case.", "Checks API removal and absence of the gateway, public IP and challenge storage. Does not enumerate all dependent Azure resources or inspect vault soft-deletion, secret values, certificate chains, renewal or upgrades.", "Skips at weekends unless explicitly overridden after checking certificate rate limits. A skip does not prove the lifecycle.", diff --git a/e2e_tests/resources/certificate.py b/e2e_tests/resources/certificate.py index 5d26fa9281..d520bfbd39 100644 --- a/e2e_tests/resources/certificate.py +++ b/e2e_tests/resources/certificate.py @@ -5,15 +5,19 @@ from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone import os +import json +from pathlib import Path +import shlex import re -from uuid import UUID +from uuid import UUID, uuid4 from cryptography import x509 from cryptography.hazmat.primitives import hashes from e2e_tests import config from e2e_tests.cloud import get_cloud -from e2e_tests.resources.sql_probe import arm_client +from e2e_tests.resources.sql_probe import arm_client, delete_after, COMPUTE_API +from e2e_tests.resources.certificate_probe import PREFIX NETWORK_API = "2024-05-01" @@ -22,7 +26,7 @@ CHECK_TIMEOUT_SECONDS = 5 * 60 -def certificate_metadata(gateway, public_ip, group, tre_id, resource_id): +def certificate_reference(gateway, public_ip, group, tre_id, resource_id): """Return only public metadata after checking resource ownership and identity.""" expected_tags = {"tre_id": tre_id, "tre_shared_service_id": resource_id} for resource, suffix in ( @@ -43,7 +47,17 @@ def certificate_metadata(gateway, public_ip, group, tre_id, resource_id): assert re.fullmatch( rf"https://kv-{re.escape(tre_id)}\.vault\.azure\.net/secrets/nexus-ssl/[a-fA-F0-9]+", secret_id ), "The gateway certificate must reference the named certificate in the core vault" - data = base64.b64decode(properties["publicCertData"], validate=True) + return fqdn, properties + + +def certificate_metadata(gateway, public_ip, group, tre_id, resource_id, public_cert_data=None): + fqdn, properties = certificate_reference(gateway, public_ip, group, tre_id, resource_id) + encoded = properties.get("publicCertData") + if encoded is None or encoded == "": + encoded = public_cert_data + if not encoded: + raise ValueError("No public certificate data was returned for the gateway reference") + data = base64.b64decode(encoded, validate=True) cert = ( x509.load_pem_x509_certificate(data) if data.startswith(b"-----BEGIN CERTIFICATE-----") @@ -65,10 +79,14 @@ def reuse_query(resource_id, digest): return f""" let received = AppTraces | where Message contains 'Message received for resource_id={resource_id},' -| where isnotempty(OperationId) -| distinct OperationId; +| where isnotempty(OperationId) and isnotempty(ParentId) +| distinct OperationId, ParentId; +let installations = AppTraces +| where Message contains 'install action starting for {resource_id}...' +| distinct OperationId, ParentId; AppTraces -| where OperationId in (received) +| join kind=leftsemi (received) on OperationId, ParentId +| join kind=leftsemi (installations) on OperationId, ParentId | summarize reused=countif(Message contains 'Reusing certificate nexus-ssl, SHA-256 fingerprint {digest}; no ACME request required'), issued=countif(Message contains 'Certificate issuance required:'), @@ -104,6 +122,7 @@ def __init__(self, arm, subscription_id, tre_id): storage = f"stwebcerts{tre_id}".replace("-", "") self.storage_id = f"{self.group}/providers/Microsoft.Storage/storageAccounts/{storage}" self.logs_id = None + self.probe_vm = None async def prepare(self): workspace = await self.arm.request( @@ -112,6 +131,77 @@ async def prepare(self): self.logs_id = str(UUID(workspace["properties"]["customerId"])) # Check query access before recovering or creating any shared service. await self.query(reuse_query("00000000-0000-0000-0000-000000000000", "0" * 64)) + await self.prepare_probe() + + async def prepare_probe(self): + vmss_id = f"{self.group}/providers/Microsoft.Compute/virtualMachineScaleSets/vmss-rp-porter-{self.tre_id}" + vmss = await self.arm.request("GET", vmss_id, COMPUTE_API) + assert vmss.get("id", "").lower() == vmss_id.lower(), "Unexpected resource-processor scale set" + assert vmss.get("tags", {}).get("tre_id") == self.tre_id, "Resource processor belongs to another TRE" + identity_id = f"{self.group}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vmss-{self.tre_id}" + identities = vmss.get("identity", {}).get("userAssignedIdentities", {}) + identities = [value for key, value in identities.items() if key.lower() == identity_id.lower()] + assert len(identities) == 1, "Expected the TRE resource-processor identity" + client_id = str(UUID(identities[0]["clientId"])) + instances = await self.arm.request("GET", vmss_id + "/virtualMachines", COMPUTE_API) + for instance in instances.get("value", []): + vm_id = instance.get("id", "") + if not re.fullmatch(re.escape(vmss_id) + r"/virtualMachines/[0-9]+", vm_id, re.I): + raise ValueError("Unexpected resource-processor VM") + view = await self.arm.request("GET", vm_id + "/instanceView", COMPUTE_API) + if any(item.get("code") == "PowerState/running" for item in view.get("statuses", [])): + self.probe_vm = (vm_id, vmss["location"], client_id) + return + raise RuntimeError("No running resource-processor VM is available for public certificate reads") + + async def public_certificate(self, secret_id): + if not re.fullmatch( + rf"https://kv-{re.escape(self.tre_id)}\.vault\.azure\.net/secrets/nexus-ssl/[a-fA-F0-9]+", secret_id + ): + raise ValueError("Unexpected certificate reference") + certificate_url = secret_id.replace("/secrets/", "/certificates/") + vm_id, location, client_id = self.probe_vm + script = Path(__file__).with_name("certificate_probe.py").read_text() + script = f"python3 - {shlex.quote(certificate_url)} {shlex.quote(client_id)} <<'PY'\n{script}\nPY" + command = f"{vm_id}/runCommands/certificate-evidence-{uuid4().hex}" + # Delete only this temporary command, including after timeout or cancellation. + async with delete_after(self.arm, command, COMPUTE_API): + async with asyncio.timeout(180): + await self.arm.request( + "PUT", + command, + COMPUTE_API, + body={ + "location": location, + "properties": { + "source": {"script": script}, + "timeoutInSeconds": 120, + "treatFailureAsDeploymentFailure": True, + }, + }, + ) + while True: + result = await self.arm.request("GET", command, COMPUTE_API, expand=True) + properties = result.get("properties", {}) + view = properties.get("instanceView", {}) + state = view.get("executionState") + if properties.get("provisioningState") == "Failed" or state in ("Failed", "Canceled", "TimedOut"): + raise RuntimeError("Public certificate probe failed") + if state == "Succeeded": + if view.get("exitCode") != 0: + raise RuntimeError("Public certificate probe returned a non-zero exit code") + lines = [ + line[len(PREFIX) :] + for line in view.get("output", "").splitlines() + if line.startswith(PREFIX) + ] + if len(lines) != 1: + raise RuntimeError("Public certificate probe returned incomplete evidence") + certificate = json.loads(lines[0]) + if certificate.get("id") != certificate_url or certificate.get("sid") != secret_id: + raise ValueError("Public certificate probe returned a different certificate version") + return certificate["cer"] + await asyncio.sleep(5) async def query(self, query): token = await asyncio.to_thread(self.arm.credential.get_token, "https://api.loganalytics.io/.default") @@ -134,7 +224,11 @@ async def snapshot(self, resource_path): break await asyncio.sleep(10) public_ip = await self.arm.request("GET", self.ip_id, NETWORK_API) - evidence = certificate_metadata(gateway, public_ip, self.group, self.tre_id, resource_id) + _, properties = certificate_reference(gateway, public_ip, self.group, self.tre_id, resource_id) + encoded = properties.get("publicCertData") + if encoded is None or encoded == "": + encoded = await self.public_certificate(properties["keyVaultSecretId"]) + evidence = certificate_metadata(gateway, public_ip, self.group, self.tre_id, resource_id, encoded) return {**evidence, "gateway_stopped": True} async def confirm_reuse(self, resource_path, digest): diff --git a/e2e_tests/resources/certificate_probe.py b/e2e_tests/resources/certificate_probe.py new file mode 100644 index 0000000000..ce79025b99 --- /dev/null +++ b/e2e_tests/resources/certificate_probe.py @@ -0,0 +1,68 @@ +"""Read one public certificate through the resource processor's private network.""" + +import base64 +import http.client +import json +import re +import sys +from urllib.parse import urlencode, urlsplit +from uuid import UUID + + +PREFIX = "TRE_PUBLIC_CERTIFICATE=" + + +def read_json(connection, path, headers): + try: + connection.request("GET", path, headers=headers) + response = connection.getresponse() + if response.status != 200: + raise RuntimeError(f"Public certificate probe received HTTP {response.status}") + content = response.read(65537) + if len(content) > 65536: + raise ValueError("Public certificate probe response exceeds its limit") + return json.loads(content) + finally: + connection.close() + + +def read_certificate(certificate_url, client_id): + if not re.fullmatch( + r"https://kv-[a-z0-9][a-z0-9-]{1,20}\.vault\.azure\.net/certificates/nexus-ssl/[a-fA-F0-9]+", + certificate_url, + ): + raise ValueError("Expected a versioned public certificate in the core vault") + client_id = str(UUID(client_id)) + query = urlencode({"api-version": "2018-02-01", "resource": "https://vault.azure.net", "client_id": client_id}) + token = read_json( + http.client.HTTPConnection("169.254.169.254", timeout=10), + "/metadata/identity/oauth2/token?" + query, + {"Metadata": "true"}, + )["access_token"] + parsed = urlsplit(certificate_url) + certificate = read_json( + http.client.HTTPSConnection(parsed.hostname, timeout=30), + parsed.path + "?api-version=7.4", + {"Authorization": "Bearer " + token}, + ) + secret_id = certificate_url.replace("/certificates/", "/secrets/") + if certificate.get("id") != certificate_url or certificate.get("sid") != secret_id: + raise ValueError("Public certificate version does not match the gateway reference") + if certificate.get("attributes", {}).get("enabled") is not True: + raise ValueError("The referenced certificate is disabled") + base64.b64decode(certificate["cer"], validate=True) + result = {key: certificate[key] for key in ("id", "sid", "cer")} + output = PREFIX + json.dumps(result, separators=(",", ":")) + # Managed Run Command retains only the final 4 KiB of output. + if len(output.encode()) > 3500: + raise ValueError("Public certificate exceeds the Run Command output limit") + return output + + +if __name__ == "__main__": + try: + print(read_certificate(sys.argv[1], sys.argv[2])) + except Exception: + # Do not print credentials, response bodies or exception values. + print("Public certificate probe failed", file=sys.stderr) + sys.exit(1) diff --git a/e2e_tests/unit_tests/test_certificate_evidence.py b/e2e_tests/unit_tests/test_certificate_evidence.py index b6ea80cb66..9024c6d165 100644 --- a/e2e_tests/unit_tests/test_certificate_evidence.py +++ b/e2e_tests/unit_tests/test_certificate_evidence.py @@ -120,8 +120,10 @@ def test_rejects_unrelated_resource_and_vault_references(self): def test_query_correlates_resource_trace_and_returns_only_counts(self): query = certs.reuse_query(RESOURCE, "a" * 64) self.assertIn(f"Message received for resource_id={RESOURCE},", query) - self.assertIn("where OperationId in (received)", query) self.assertIn("summarize", query) + self.assertIn("join kind=leftsemi (received) on OperationId, ParentId", query) + self.assertIn("join kind=leftsemi (installations) on OperationId, ParentId", query) + self.assertIn(f"install action starting for {RESOURCE}...", query) self.assertIn("no ACME request required", query) with self.assertRaises(ValueError): certs.reuse_query("'; AppTraces", "a" * 64) @@ -230,7 +232,57 @@ def handle(request): async def test_prepare_checks_logs_access_before_creating_resources(self): self.arm.request.return_value = {"properties": {"customerId": WORKSPACE}} self.checks.query = AsyncMock(return_value={"reused": 0, "issued": 0, "completed": 0}) - await self.checks.prepare() + with patch.object(self.checks, "prepare_probe", AsyncMock()) as prepare_probe: + await self.checks.prepare() + prepare_probe.assert_awaited_once() self.assertEqual(self.checks.logs_id, WORKSPACE) self.checks.query.assert_awaited_once() self.assertEqual(self.arm.request.call_args.args[0], "GET") + + +class CertificateFallbackTests(unittest.IsolatedAsyncioTestCase): + async def test_missing_or_empty_gateway_data_uses_the_exact_vault_reference(self): + for missing in (None, ""): + with self.subTest(missing=missing): + gateway, ip, digest = resources() + properties = gateway["properties"]["sslCertificates"][0]["properties"] + data = properties.pop("publicCertData") + if missing is not None: + properties["publicCertData"] = missing + arm = SimpleNamespace(request=AsyncMock(side_effect=[gateway, ip])) + checks = certs.CertificateChecks(arm, SUBSCRIPTION, TRE_ID) + checks.public_certificate = AsyncMock(return_value=data) + result = await checks.snapshot(f"/shared-services/{RESOURCE}") + self.assertEqual(result, {"hostname": HOSTNAME, "sha256": digest, "gateway_stopped": True}) + checks.public_certificate.assert_awaited_once_with(properties["keyVaultSecretId"]) + + async def test_fallback_keeps_hostname_and_validity_checks(self): + for parameters in ({"hostname": "other.example.com"}, {"days": 29}): + gateway, ip, _ = resources(**parameters) + properties = gateway["properties"]["sslCertificates"][0]["properties"] + data = properties.pop("publicCertData") + arm = SimpleNamespace(request=AsyncMock(side_effect=[gateway, ip])) + checks = certs.CertificateChecks(arm, SUBSCRIPTION, TRE_ID) + checks.public_certificate = AsyncMock(return_value=data) + with self.subTest(parameters=parameters), self.assertRaises(AssertionError): + await checks.snapshot(f"/shared-services/{RESOURCE}") + + async def test_wrong_ownership_is_rejected_before_fallback(self): + gateway, ip, _ = resources() + gateway["tags"]["tre_id"] = "another-tre" + arm = SimpleNamespace(request=AsyncMock(side_effect=[gateway, ip])) + checks = certs.CertificateChecks(arm, SUBSCRIPTION, TRE_ID) + checks.public_certificate = AsyncMock() + with self.assertRaises(AssertionError): + await checks.snapshot(f"/shared-services/{RESOURCE}") + checks.public_certificate.assert_not_awaited() + + async def test_malformed_gateway_data_does_not_fall_back(self): + gateway, ip, _ = resources() + gateway["properties"]["sslCertificates"][0]["properties"]["publicCertData"] = "invalid!" + arm = SimpleNamespace(request=AsyncMock(side_effect=[gateway, ip])) + checks = certs.CertificateChecks(arm, SUBSCRIPTION, TRE_ID) + checks.public_certificate = AsyncMock() + with self.assertRaises(ValueError): + await checks.snapshot(f"/shared-services/{RESOURCE}") + checks.public_certificate.assert_not_awaited() diff --git a/e2e_tests/unit_tests/test_certificate_probe.py b/e2e_tests/unit_tests/test_certificate_probe.py new file mode 100644 index 0000000000..e6edaa7bca --- /dev/null +++ b/e2e_tests/unit_tests/test_certificate_probe.py @@ -0,0 +1,163 @@ +"""Check certificate-only reads, VM ownership and temporary command cleanup.""" + +import asyncio +from copy import deepcopy +import json +from types import SimpleNamespace +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from e2e_tests.resources import certificate as certs, certificate_probe as probe +from e2e_tests.unit_tests.test_certificate_evidence import SUBSCRIPTION, TRE_ID, GROUP, gateway_certificate + + +CLIENT_ID = "44444444-4444-4444-8444-444444444444" +CERTIFICATE_URL = f"https://kv-{TRE_ID}.vault.azure.net/certificates/nexus-ssl/abcd" +SECRET_ID = CERTIFICATE_URL.replace("/certificates/", "/secrets/") +VMSS = f"{GROUP}/providers/Microsoft.Compute/virtualMachineScaleSets/vmss-rp-porter-{TRE_ID}" +VM = VMSS + "/virtualMachines/0" +IDENTITY = f"{GROUP}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vmss-{TRE_ID}" + + +class PublicCertificateScriptTests(unittest.TestCase): + def certificate(self): + return { + "id": CERTIFICATE_URL, + "sid": SECRET_ID, + "cer": gateway_certificate()[0], + "attributes": {"enabled": True}, + "private": "omit", + } + + def test_reads_only_public_certificate_with_existing_identity(self): + certificate = self.certificate() + with patch.object(probe, "read_json", side_effect=[{"access_token": "token"}, certificate]) as read: + output = probe.read_certificate(CERTIFICATE_URL, CLIENT_ID) + saved = json.loads(output.removeprefix(probe.PREFIX)) + self.assertEqual(set(saved), {"id", "sid", "cer"}) + self.assertNotIn("token", output) + self.assertNotIn("omit", output) + self.assertEqual(read.call_args_list[0].args[0].host, "169.254.169.254") + self.assertIn(CLIENT_ID, read.call_args_list[0].args[1]) + self.assertEqual(read.call_args_list[1].args[0].host, f"kv-{TRE_ID}.vault.azure.net") + self.assertEqual(read.call_args_list[1].args[1], "/certificates/nexus-ssl/abcd?api-version=7.4") + + def test_rejects_other_hosts_secret_paths_unversioned_urls_and_invalid_identity(self): + for url, identity in ( + (CERTIFICATE_URL.replace(".azure.net", ".example.com"), CLIENT_ID), + (SECRET_ID, CLIENT_ID), + (CERTIFICATE_URL.rsplit("/", 1)[0], CLIENT_ID), + (CERTIFICATE_URL + "?other=true", CLIENT_ID), + (CERTIFICATE_URL, "not-a-uuid"), + ): + with self.subTest(url=url, identity=identity), patch.object(probe, "read_json") as read: + with self.assertRaises(ValueError): + probe.read_certificate(url, identity) + read.assert_not_called() + + def test_rejects_disabled_mismatched_or_oversize_certificates(self): + certificate = self.certificate() + for changes in ( + {"id": CERTIFICATE_URL + "f"}, + {"sid": SECRET_ID + "f"}, + {"attributes": {"enabled": False}}, + {"cer": "invalid!"}, + {"cer": "A" * 5000}, + ): + with ( + self.subTest(changes=list(changes)), + patch.object(probe, "read_json", side_effect=[{"access_token": "token"}, {**certificate, **changes}]), + ): + with self.assertRaises(ValueError): + probe.read_certificate(CERTIFICATE_URL, CLIENT_ID) + + def test_redirects_and_failed_reads_are_not_followed_or_exposed(self): + for code in (302, 403, 500): + connection = Mock() + connection.getresponse.return_value.status = code + with self.subTest(code=code), self.assertRaisesRegex(RuntimeError, f"HTTP {code}"): + probe.read_json(connection, "/certificates/nexus-ssl/abcd", {}) + connection.request.assert_called_once() + connection.getresponse.return_value.read.assert_not_called() + connection.close.assert_called_once() + + +class PublicCertificateCommandTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.arm = SimpleNamespace(request=AsyncMock(), delete=AsyncMock()) + self.checks = certs.CertificateChecks(self.arm, SUBSCRIPTION, TRE_ID) + self.checks.probe_vm = (VM, "switzerlandnorth", CLIENT_ID) + self.cer = gateway_certificate()[0] + + def result(self, **changes): + result = { + "executionState": "Succeeded", + "exitCode": 0, + "output": probe.PREFIX + json.dumps({"id": CERTIFICATE_URL, "sid": SECRET_ID, "cer": self.cer}), + } + result.update(changes) + return {"properties": {"instanceView": result}} + + async def test_success_reads_exact_version_and_deletes_only_its_command(self): + self.arm.request.side_effect = [{}, self.result()] + result = await self.checks.public_certificate(SECRET_ID) + self.assertEqual(result, self.cer) + put = self.arm.request.call_args_list[0] + self.assertEqual(put.args[0], "PUT") + self.assertTrue(put.args[1].startswith(VM + "/runCommands/certificate-evidence-")) + script = put.kwargs["body"]["properties"]["source"]["script"] + self.assertIn(CERTIFICATE_URL, script) + self.assertIn(CLIENT_ID, script) + self.arm.delete.assert_awaited_once_with(put.args[1], certs.COMPUTE_API) + + async def test_failed_partial_or_wrong_version_evidence_still_cleans_up(self): + for result in ( + self.result(executionState="Failed"), + self.result(exitCode=1), + self.result(output=""), + self.result( + output=probe.PREFIX + json.dumps({"id": CERTIFICATE_URL + "f", "sid": SECRET_ID, "cer": self.cer}) + ), + ): + self.arm.delete.reset_mock() + self.arm.request.side_effect = [{}, result] + with ( + self.subTest(result=result["properties"]["instanceView"]["executionState"]), + self.assertRaises((RuntimeError, ValueError)), + ): + await self.checks.public_certificate(SECRET_ID) + self.arm.delete.assert_awaited_once() + + async def test_cancellation_after_put_still_deletes_command(self): + self.arm.request.side_effect = [{}, asyncio.CancelledError()] + with self.assertRaises(asyncio.CancelledError): + await self.checks.public_certificate(SECRET_ID) + self.arm.delete.assert_awaited_once() + + async def test_rejects_unrelated_vault_before_creating_command(self): + with self.assertRaises(ValueError): + await self.checks.public_certificate(SECRET_ID.replace(TRE_ID, "another-tre")) + self.arm.request.assert_not_awaited() + + async def test_prepare_requires_owned_running_vm_and_expected_identity(self): + vmss = { + "id": VMSS, + "location": "switzerlandnorth", + "tags": {"tre_id": TRE_ID}, + "identity": {"userAssignedIdentities": {IDENTITY: {"clientId": CLIENT_ID}}}, + } + instances = {"value": [{"id": VM}]} + running = {"statuses": [{"code": "PowerState/running"}]} + self.arm.request.side_effect = [vmss, instances, running] + await self.checks.prepare_probe() + self.assertEqual(self.checks.probe_vm, (VM, "switzerlandnorth", CLIENT_ID)) + for changed in ({**vmss, "tags": {"tre_id": "other"}}, {**vmss, "identity": {}}): + self.arm.request.side_effect = [changed] + with self.assertRaises(AssertionError): + await self.checks.prepare_probe() + self.arm.request.side_effect = [deepcopy(vmss), {"value": [{"id": VM.replace("/0", "/../another")}]}] + with self.assertRaises(ValueError): + await self.checks.prepare_probe() + self.arm.request.side_effect = [vmss, instances, {"statuses": [{"code": "PowerState/deallocated"}]}] + with self.assertRaisesRegex(RuntimeError, "No running"): + await self.checks.prepare_probe() From 2ce0cdffbbb3b8e3dbf349c74cb323a3b805d0b2 Mon Sep 17 00:00:00 2001 From: Jay W Date: Thu, 15 Oct 2026 17:43:19 +0100 Subject: [PATCH 16/17] Allow slower Azure health checks in smoke tests (#5119) The smoke health request timed out after five seconds in run 38769684961. Application Insights recorded HTTP 200 after 15.399 seconds for that request. Set an explicit 30-second HTTP timeout and retain all service-health assertions. This changes the test wait, not the API health implementation. Failure: https://github.com/microsoft/AzureTRE/actions/runs/38769684961/job/116542255620 Validation: the actual smoke test accepts a delayed healthy local response and rejects an unhealthy service and HTTP 503. Applicable pre-commit checks pass. Live Azure validation remains pending. --- CHANGELOG.md | 1 + e2e_tests/test_provisioned_health_api.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2496fe247f..f36075a207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Set an explicit 30-second timeout for the smoke health request so Azure health checks have time to complete. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Read missing gateway certificate metadata through the resource processor and isolate reuse evidence by installation message span. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Reject invalid or duplicate SHA arguments before starting shared-service tests. (`build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Read the shared-service template response correctly before certificate lifecycle validation. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) diff --git a/e2e_tests/test_provisioned_health_api.py b/e2e_tests/test_provisioned_health_api.py index 92636d12dc..02a4df307a 100644 --- a/e2e_tests/test_provisioned_health_api.py +++ b/e2e_tests/test_provisioned_health_api.py @@ -12,7 +12,8 @@ async def test_health() -> None: async with AsyncClient(verify=False) as client: url = f"{config.TRE_URL}{strings.API_HEALTH}" - response = await client.get(url) + # Azure management health checks can exceed HTTPX's five-second default. + response = await client.get(url, timeout=30) assert response.status_code == 200 assert response.json()["services"] == [ {"service": "Cosmos DB", "status": "OK", "message": ""}, From 585ecb26269e7b468130383851c5d72a0536c7b7 Mon Sep 17 00:00:00 2001 From: Jay W Date: Fri, 16 Oct 2026 14:21:45 +0100 Subject: [PATCH 17/17] Set an explicit admin username for Linux VM validation (#5145) The CI owner is a service principal. Supply the Linux admin username so the bundle does not resolve that owner through azuread_user. Observed in https://github.com/microsoft/AzureTRE/actions/runs/38896970592/job/116960509490 --- CHANGELOG.md | 1 + e2e_tests/test_guacamole_linuxvm.py | 2 ++ e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f36075a207..1ede6c98d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ See [Legacy Airlock & migration](docs/azure-tre-overview/airlock.md#legacy-airlo * Add weekly and manual builds for Dockerfiles and Porter bundles, with selected pull request builds. Retry a failed target once and report final results. ([#5088](https://github.com/microsoft/AzureTRE/issues/5088)) BUG FIXES: +* Set an explicit admin username in Linux VM validation so the bundle does not look up the CI service principal as an Entra user. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Set an explicit 30-second timeout for the smoke health request so Azure health checks have time to complete. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Read missing gateway certificate metadata through the resource processor and isolate reuse evidence by installation message span. ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) * Reject invalid or duplicate SHA arguments before starting shared-service tests. (`build-scripts` 1.1.4) ([#5255](https://github.com/microsoft/AzureTRE/pull/5255)) diff --git a/e2e_tests/test_guacamole_linuxvm.py b/e2e_tests/test_guacamole_linuxvm.py index 6b37a6353b..1cc4ada88f 100644 --- a/e2e_tests/test_guacamole_linuxvm.py +++ b/e2e_tests/test_guacamole_linuxvm.py @@ -23,6 +23,8 @@ async def test_create_guacamole_linux_vm(setup_test_workspace_and_guacamole_serv "description": "Verify jammy package installation and cloud-init completion", "os_image": "Ubuntu 22.04 LTS", "vm_size": "2 CPU | 8GB RAM", + # CI uses a service principal, so the bundle cannot derive a username from its owner. + "admin_username": "researcher", }, } diff --git a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py index c174446ba4..e615f7a047 100644 --- a/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py +++ b/e2e_tests/unit_tests/test_guacamole_linuxvm_cleanup.py @@ -107,9 +107,11 @@ def assert_deleted(self, failed_disable=False, failed_create=False): ) self.assertEqual(self.requests[-2].url.path, f"/api{self.resource_path}") - async def test_success_cleans_up_vm_in_reused_service(self): + async def test_success_uses_explicit_username_and_cleans_up_vm(self): self.mock_api(strings.RESOURCE_STATUS_DEPLOYED) await linuxvm.test_create_guacamole_linux_vm(self.fixture, True) + payload = json.loads(self.requests[0].content) + self.assertEqual(payload["properties"]["admin_username"], "researcher") self.assert_deleted() async def test_failed_bootstrap_cleans_up_after_failed_disable(self):