From 2ba561a59b5f69bdb290fc79fe374a2f2c5d9069 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 10:28:41 -0700 Subject: [PATCH 01/10] CI: add runner certificate trust store probe workflow Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 441 ++++++++++++++++++++++ ci/tools/compute_ci_plan.py | 6 + ci/tools/runner_trust_probe.ps1 | 103 +++++ ci/tools/runner_trust_probe.sh | 141 +++++++ ci/tools/tests/test_compute_ci_plan.py | 4 + ci/tools/tests/test_runner_trust_probe.py | 126 +++++++ 6 files changed, 821 insertions(+) create mode 100644 .github/workflows/runner-trust-probe.yml create mode 100644 ci/tools/runner_trust_probe.ps1 create mode 100755 ci/tools/runner_trust_probe.sh create mode 100644 ci/tools/tests/test_runner_trust_probe.py diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml new file mode 100644 index 00000000000..5ce40784c1d --- /dev/null +++ b/.github/workflows/runner-trust-probe.yml @@ -0,0 +1,441 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Report-only probe of OS and bundled Node.js certificate stores across hosted +# and self-hosted runner labels for the eMudhra/emSign CA incident. + +name: "CI: Runner trust probe" + +on: + schedule: + # Weekly on Mondays at 06:23 UTC + - cron: "23 6 * * 1" + workflow_dispatch: + inputs: + label-filter: + description: "Optional regex to filter runner labels (e.g. '^(ubuntu-latest|linux-amd64-cpu8)$'; empty probes all)" + required: false + default: "" + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash --noprofile --norc -xeuo pipefail {0} + +jobs: + compute-matrix: + name: Compute runner label sets + if: ${{ github.repository_owner == 'nvidia' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + linux_host: ${{ steps.compute.outputs.linux_host }} + linux_gpu: ${{ steps.compute.outputs.linux_gpu }} + windows: ${{ steps.compute.outputs.windows }} + has_linux_host: ${{ steps.compute.outputs.has_linux_host }} + has_linux_gpu: ${{ steps.compute.outputs.has_linux_gpu }} + has_windows: ${{ steps.compute.outputs.has_windows }} + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Compute runner labels + id: compute + env: + LABEL_FILTER: ${{ inputs.label-filter || '' }} + run: | + matrix_json="$(yq -o json '.' ci/test-matrix.yml)" + + linux_host="$(jq -cn --arg filter "${LABEL_FILTER}" ' + [ + "ubuntu-latest", + "linux-amd64-cpu8", + "linux-arm64-cpu8", + "linux-amd64-cpu4" + ] + | unique + | if $filter != "" then map(select(test($filter))) else . end + ')" + + linux_gpu="$(jq -cn --argjson m "${matrix_json}" --arg filter "${LABEL_FILTER}" ' + [ + ($m.linux["pull-request"][]?, $m.linux["nightly"][]?) + | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" + ] + | unique + | if $filter != "" then map(select(test($filter))) else . end + ')" + + windows="$(jq -cn --argjson m "${matrix_json}" --arg filter "${LABEL_FILTER}" ' + ( + [ + "windows-latest", + "windows-2022", + "windows-11-arm" + ] + [ + ($m.windows["pull-request"][]?, $m.windows["nightly"][]?) + | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" + ] + ) + | unique + | if $filter != "" then map(select(test($filter))) else . end + ')" + + echo "linux_host=${linux_host}" | tee -a "${GITHUB_OUTPUT}" + echo "linux_gpu=${linux_gpu}" | tee -a "${GITHUB_OUTPUT}" + echo "windows=${windows}" | tee -a "${GITHUB_OUTPUT}" + echo "has_linux_host=$(jq 'length > 0' <<< "${linux_host}")" | tee -a "${GITHUB_OUTPUT}" + echo "has_linux_gpu=$(jq 'length > 0' <<< "${linux_gpu}")" | tee -a "${GITHUB_OUTPUT}" + echo "has_windows=$(jq 'length > 0' <<< "${windows}")" | tee -a "${GITHUB_OUTPUT}" + + probe-linux-host: + name: Probe (${{ matrix.label }}) + needs: compute-matrix + if: ${{ github.repository_owner == 'nvidia' && needs.compute-matrix.outputs.has_linux_host == 'true' }} + strategy: + fail-fast: false + matrix: + label: ${{ fromJSON(needs.compute-matrix.outputs.linux_host) }} + runs-on: ${{ matrix.label }} + timeout-minutes: 15 + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Probe OS trust store + id: os-probe + continue-on-error: ${{ matrix.label == 'ubuntu-latest' }} + env: + TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}" + run: ./ci/tools/runner_trust_probe.sh + + - name: Probe OS trust store after ca-certificates upgrade (ubuntu-latest) + id: os-probe-upgraded + if: ${{ always() && matrix.label == 'ubuntu-latest' }} + continue-on-error: true + env: + TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)" + run: | + sudo apt-get update + sudo apt-get install --only-upgrade -y ca-certificates + ./ci/tools/runner_trust_probe.sh + + - name: Probe Node 20 action runtime (informational) + if: ${{ always() }} + continue-on-error: true + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) + with: + script: | + const msg = `Node 20 action runtime resolved to \`${process.version}\` (\`${process.execPath}\`)`; + core.info(msg); + await core.summary + .addHeading('Node 20 action runtime (informational)', 3) + .addRaw(msg + '\n\n') + .write(); + + - name: Probe bundled Node runtime and trust store + id: node-probe + if: ${{ always() }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const https = require('https'); + const version = process.version; + const execPath = process.execPath; + const nodeOptions = process.env.NODE_OPTIONS || ''; + const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; + const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; + + const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); + const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; + const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); + + const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; + const probeEndpoint = () => new Promise((resolve) => { + const req = https.get(endpoint, { timeout: 15000 }, (res) => { + res.resume(); + resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); + }); + req.on('timeout', () => { + req.destroy(new Error('Request timed out')); + }); + req.on('error', (err) => { + const code = err.code || ''; + const msg = `${code} ${err.message || ''}`.trim(); + if (/CERT|UNABLE_TO_VERIFY|SELF_SIGNED|DEPTH_ZERO|SSL|TLS/i.test(msg)) { + resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); + } else { + resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); + } + }); + }); + + const tlsResult = await probeEndpoint(); + const rows = [ + ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require \`>= v24.22.1\` on Node 24; \`execPath=${execPath}\`)`], + ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], + ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], + ]; + + await core.summary + .addHeading('Node runtime trust store', 3) + .addTable([ + [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], + ...rows, + ]) + .write(); + + if (!versionOk || tlsResult.status === 'FAIL') { + core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); + } + + - name: Check probe outcomes + if: ${{ always() }} + run: | + if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then + echo "::error::Initial OS trust store probe failed" >&2 + exit 1 + fi + if [[ "${{ matrix.label }}" == "ubuntu-latest" && "${{ steps.os-probe-upgraded.outcome }}" != "success" ]]; then + echo "::error::Post-upgrade OS trust store probe failed on ubuntu-latest" >&2 + exit 1 + fi + if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then + echo "::error::Node runtime trust store probe failed" >&2 + exit 1 + fi + + probe-linux-gpu: + name: Probe (${{ matrix.label }}) + needs: compute-matrix + if: ${{ github.repository_owner == 'nvidia' && needs.compute-matrix.outputs.has_linux_gpu == 'true' }} + strategy: + fail-fast: false + matrix: + label: ${{ fromJSON(needs.compute-matrix.outputs.linux_gpu) }} + runs-on: ${{ matrix.label }} + timeout-minutes: 15 + # Match test-wheel-linux.yml container setup so the probe inspects the exact + # container environment that Linux GPU test jobs see. + container: + options: -u root --security-opt seccomp=unconfined --shm-size 16g + image: ubuntu:24.04 + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Setup proxy cache + uses: nv-gha-runners/setup-proxy-cache@main + continue-on-error: true + + - name: Install container CA certificates and probe tools + uses: ./.github/actions/install_unix_deps + continue-on-error: false + with: + dependencies: "ca-certificates curl openssl" + dependent_exes: "curl openssl" + + - name: Probe container OS trust store + id: os-probe + continue-on-error: true + env: + TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)" + run: ./ci/tools/runner_trust_probe.sh + + - name: Probe Node 20 action runtime (informational) + if: ${{ always() }} + continue-on-error: true + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) + with: + script: | + const msg = `Node 20 action runtime resolved to \`${process.version}\` (\`${process.execPath}\`)`; + core.info(msg); + await core.summary + .addHeading('Node 20 action runtime (informational)', 3) + .addRaw(msg + '\n\n') + .write(); + + - name: Probe bundled Node runtime and trust store + id: node-probe + if: ${{ always() }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const https = require('https'); + const version = process.version; + const execPath = process.execPath; + const nodeOptions = process.env.NODE_OPTIONS || ''; + const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; + const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; + + const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); + const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; + const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); + + const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; + const probeEndpoint = () => new Promise((resolve) => { + const req = https.get(endpoint, { timeout: 15000 }, (res) => { + res.resume(); + resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); + }); + req.on('timeout', () => { + req.destroy(new Error('Request timed out')); + }); + req.on('error', (err) => { + const code = err.code || ''; + const msg = `${code} ${err.message || ''}`.trim(); + if (/CERT|UNABLE_TO_VERIFY|SELF_SIGNED|DEPTH_ZERO|SSL|TLS/i.test(msg)) { + resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); + } else { + resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); + } + }); + }); + + const tlsResult = await probeEndpoint(); + const rows = [ + ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require \`>= v24.22.1\` on Node 24; \`execPath=${execPath}\`)`], + ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], + ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], + ]; + + await core.summary + .addHeading('Node runtime trust store', 3) + .addTable([ + [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], + ...rows, + ]) + .write(); + + if (!versionOk || tlsResult.status === 'FAIL') { + core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); + } + + - name: Check probe outcomes + if: ${{ always() }} + run: | + if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then + echo "::error::Container OS trust store probe failed" >&2 + exit 1 + fi + if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then + echo "::error::Node runtime trust store probe failed" >&2 + exit 1 + fi + + probe-windows: + name: Probe (${{ matrix.label }}) + needs: compute-matrix + if: ${{ github.repository_owner == 'nvidia' && needs.compute-matrix.outputs.has_windows == 'true' }} + strategy: + fail-fast: false + matrix: + label: ${{ fromJSON(needs.compute-matrix.outputs.windows) }} + runs-on: ${{ matrix.label }} + timeout-minutes: 15 + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Probe Windows OS trust store + id: os-probe + continue-on-error: true + shell: pwsh + run: ./ci/tools/runner_trust_probe.ps1 + + - name: Probe Node 20 action runtime (informational) + if: ${{ always() }} + continue-on-error: true + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) + with: + script: | + const msg = `Node 20 action runtime resolved to \`${process.version}\` (\`${process.execPath}\`)`; + core.info(msg); + await core.summary + .addHeading('Node 20 action runtime (informational)', 3) + .addRaw(msg + '\n\n') + .write(); + + - name: Probe bundled Node runtime and trust store + id: node-probe + if: ${{ always() }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const https = require('https'); + const version = process.version; + const execPath = process.execPath; + const nodeOptions = process.env.NODE_OPTIONS || ''; + const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; + const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; + + const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); + const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; + const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); + + const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; + const probeEndpoint = () => new Promise((resolve) => { + const req = https.get(endpoint, { timeout: 15000 }, (res) => { + res.resume(); + resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); + }); + req.on('timeout', () => { + req.destroy(new Error('Request timed out')); + }); + req.on('error', (err) => { + const code = err.code || ''; + const msg = `${code} ${err.message || ''}`.trim(); + if (/CERT|UNABLE_TO_VERIFY|SELF_SIGNED|DEPTH_ZERO|SSL|TLS/i.test(msg)) { + resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); + } else { + resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); + } + }); + }); + + const tlsResult = await probeEndpoint(); + const rows = [ + ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require \`>= v24.22.1\` on Node 24; \`execPath=${execPath}\`)`], + ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], + ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], + ]; + + await core.summary + .addHeading('Node runtime trust store', 3) + .addTable([ + [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], + ...rows, + ]) + .write(); + + if (!versionOk || tlsResult.status === 'FAIL') { + core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); + } + + - name: Check probe outcomes + if: ${{ always() }} + run: | + if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then + echo "::error::Windows OS trust store probe failed" >&2 + exit 1 + fi + if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then + echo "::error::Node runtime trust store probe failed" >&2 + exit 1 + fi diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py index ea03abc31a9..9042418a167 100644 --- a/ci/tools/compute_ci_plan.py +++ b/ci/tools/compute_ci_plan.py @@ -37,10 +37,14 @@ IGNORED_SUFFIXES = {".md", ".svg"} IGNORED_PATHS = { ".coveragerc", + ".github/workflows/runner-trust-probe.yml", ".gitignore", ".pre-commit-config.yaml", ".spdx-ignore", "LICENSE", + "ci/tools/runner_trust_probe.ps1", + "ci/tools/runner_trust_probe.sh", + "ci/tools/tests/test_runner_trust_probe.py", "context7.json", "greptile.json", "ruff.toml", @@ -86,6 +90,8 @@ def compute_workplan( if path_parts[0] == "ci" or ( len(path_parts) >= 2 and path_parts[:2] in {(".github", "actions"), (".github", "workflows")} ): + if path in IGNORED_PATHS: + continue force_all = True break diff --git a/ci/tools/runner_trust_probe.ps1 b/ci/tools/runner_trust_probe.ps1 new file mode 100644 index 00000000000..23a522e27a1 --- /dev/null +++ b/ci/tools/runner_trust_probe.ps1 @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# runner_trust_probe.ps1 -- probe the Windows certificate store on a CI runner +# for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3). +# +# Optional environment overrides for local testing: +# TRUST_PROBE_TEST_ENDPOINT HTTPS endpoint to test (default: https://dv-g3b-emudhra.we-have-a-problem.com/) +# TRUST_PROBE_LABEL Heading label for GITHUB_STEP_SUMMARY + +$ErrorActionPreference = 'Stop' + +$EmSignG3Sha256 = '86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B' +$EmSignG3Hex = $EmSignG3Sha256.Replace(':', '') +$TestEndpoint = if ($null -ne $env:TRUST_PROBE_TEST_ENDPOINT) { $env:TRUST_PROBE_TEST_ENDPOINT } else { 'https://dv-g3b-emudhra.we-have-a-problem.com/' } +$ProbeLabel = if ($env:TRUST_PROBE_LABEL) { $env:TRUST_PROBE_LABEL } else { 'Windows OS trust store' } +$SummaryFile = $env:GITHUB_STEP_SUMMARY + +function Append-Summary([string]$Line) { + if ($SummaryFile) { + Add-Content -Path $SummaryFile -Value $Line -Encoding utf8 + } +} + +function Record-Check([string]$Name, [string]$Status, [string]$Details) { + Write-Output "[$Status] ${Name}: $Details" + Append-Summary "| $Name | **$Status** | $Details |" +} + +$GatingFailures = 0 + +Append-Summary "### $ProbeLabel" +Append-Summary "" +Append-Summary "| Check | Status | Details |" +Append-Summary "| --- | --- | --- |" + +# 1. Root anchor not trusted in Cert:\LocalMachine\Root (or present in Disallowed) (gating) +$sha256 = [System.Security.Cryptography.SHA256]::Create() +try { + $rootCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Root' -ErrorAction Stop) + $inRoot = $false + foreach ($cert in $rootCerts) { + $hashBytes = $sha256.ComputeHash($cert.RawData) + $hex = [System.BitConverter]::ToString($hashBytes).Replace('-', '').ToUpperInvariant() + if ($hex -eq $EmSignG3Hex) { + $inRoot = $true + break + } + } + + $inDisallowed = $false + if (Test-Path -Path 'Cert:\LocalMachine\Disallowed') { + $disallowedCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Disallowed' -ErrorAction SilentlyContinue) + foreach ($cert in $disallowedCerts) { + $hashBytes = $sha256.ComputeHash($cert.RawData) + $hex = [System.BitConverter]::ToString($hashBytes).Replace('-', '').ToUpperInvariant() + if ($hex -eq $EmSignG3Hex) { + $inDisallowed = $true + break + } + } + } + + $storeSummary = "LocalMachine\Root=$inRoot ($($rootCerts.Count) certs), LocalMachine\Disallowed=$inDisallowed" + if ((-not $inRoot) -or $inDisallowed) { + Record-Check 'Root anchor not trusted' 'PASS' $storeSummary + } else { + Record-Check 'Root anchor not trusted' 'FAIL' "Present in LocalMachine\Root and not in Disallowed ($storeSummary)" + $GatingFailures++ + } +} finally { + $sha256.Dispose() +} + +# 2. OS-store client (Invoke-WebRequest) rejects the test endpoint (only a successful connection fails the job) +if ([string]::IsNullOrEmpty($TestEndpoint)) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'SKIP' 'Skipped via empty `TRUST_PROBE_TEST_ENDPOINT`' +} else { + $progTmp = $ProgressPreference + $ProgressPreference = 'SilentlyContinue' + try { + $null = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -TimeoutSec 15 + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded (certificate was trusted)" + $GatingFailures++ + } catch { + $msg = ($_.Exception.ToString() -replace '[\r\n]+', ' ' -replace '\|', '\|') + if ($msg -match 'AuthenticationException|TrustFailure|SSL|TLS|certificate|Cert| remote party') { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with TLS/certificate error" + } else { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-TLS or network error: $($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|')" + } + } finally { + $ProgressPreference = $progTmp + } +} + +Append-Summary "" + +if ($GatingFailures -gt 0) { + Write-Error "${ProbeLabel}: $GatingFailures gating check(s) failed" + exit 1 +} diff --git a/ci/tools/runner_trust_probe.sh b/ci/tools/runner_trust_probe.sh new file mode 100755 index 00000000000..5db50f23d92 --- /dev/null +++ b/ci/tools/runner_trust_probe.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# runner_trust_probe.sh -- probe the OS certificate store on a Linux runner +# for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3). +# +# Optional environment overrides for local testing: +# TRUST_PROBE_CA_BUNDLE Path to CA bundle (default: /etc/ssl/certs/ca-certificates.crt) +# TRUST_PROBE_TEST_ENDPOINT HTTPS endpoint to test (default: https://dv-g3b-emudhra.we-have-a-problem.com/) +# TRUST_PROBE_LABEL Heading label for GITHUB_STEP_SUMMARY + +set -euo pipefail + +EMSIGN_G3_SHA256="${TRUST_PROBE_TARGET_SHA256:-86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B}" +CA_BUNDLE="${TRUST_PROBE_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" +TEST_ENDPOINT="${TRUST_PROBE_TEST_ENDPOINT:-https://dv-g3b-emudhra.we-have-a-problem.com/}" +PROBE_LABEL="${TRUST_PROBE_LABEL:-Linux OS trust store}" + +summary_file="${GITHUB_STEP_SUMMARY:-}" +append_summary() { + if [[ -n "${summary_file}" ]]; then + printf '%s\n' "$*" >> "${summary_file}" + fi +} + +gating_failures=0 + +append_summary "### ${PROBE_LABEL}" +append_summary "" +append_summary "| Check | Status | Details |" +append_summary "| --- | --- | --- |" + +record_check() { + local name="$1" + local status="$2" + local details="$3" + printf '[%s] %s: %s\n' "${status}" "${name}" "${details}" + append_summary "| ${name} | **${status}** | ${details} |" +} + +# 1. Root anchor absent from CA bundle (gating) +if [[ ! -f "${CA_BUNDLE}" ]]; then + record_check "Root anchor absent" "FAIL" "Bundle not found: \`${CA_BUNDLE}\`" + gating_failures=$((gating_failures + 1)) +elif ! command -v openssl >/dev/null 2>&1; then + record_check "Root anchor absent" "FAIL" "\`openssl\` not found on PATH" + gating_failures=$((gating_failures + 1)) +else + tmp_dir="$(mktemp -d)" + trap 'rm -rf "${tmp_dir}"' EXIT + + awk -v dir="${tmp_dir}" ' + /-----BEGIN CERTIFICATE-----/ { in_cert = 1; idx++; file = sprintf("%s/cert-%04d.pem", dir, idx) } + in_cert { print > file } + /-----END CERTIFICATE-----/ { in_cert = 0; close(file) } + ' "${CA_BUNDLE}" + + shopt -s nullglob + cert_files=("${tmp_dir}"/cert-*.pem) + shopt -u nullglob + + if (( ${#cert_files[@]} == 0 )); then + record_check "Root anchor absent" "FAIL" "No certificates parsed from \`${CA_BUNDLE}\`" + gating_failures=$((gating_failures + 1)) + else + found_emsign=0 + for cert_file in "${cert_files[@]}"; do + fp="$(openssl x509 -in "${cert_file}" -noout -fingerprint -sha256 2>/dev/null | sed 's/^.*=//; s/[[:space:]]//g' | tr '[:lower:]' '[:upper:]')" + if [[ "${fp}" == "${EMSIGN_G3_SHA256}" ]]; then + found_emsign=1 + break + fi + done + + if (( found_emsign == 1 )); then + record_check "Root anchor absent" "FAIL" "Found \`emSign ECC Root CA - G3\` (\`${EMSIGN_G3_SHA256}\`) in \`${CA_BUNDLE}\` (${#cert_files[@]} certs checked)" + gating_failures=$((gating_failures + 1)) + else + record_check "Root anchor absent" "PASS" "Absent from \`${CA_BUNDLE}\` (${#cert_files[@]} certs checked)" + fi + fi +fi + +# 2. ca-certificates package version (informational) +ca_pkg_ver="$(dpkg-query -W -f='${Version}' ca-certificates 2>/dev/null || echo 'not-installed')" +distro_id="unknown" +distro_ver="unknown" +if [[ -f /etc/os-release ]]; then + # shellcheck disable=SC1091 + source /etc/os-release + distro_id="${ID:-unknown}" + distro_ver="${VERSION_ID:-unknown}" +fi + +if [[ "${ca_pkg_ver}" == "not-installed" ]]; then + record_check "ca-certificates version" "INFO" "Package not installed (distro: \`${distro_id} ${distro_ver}\`)" +elif [[ "${distro_id}" == "ubuntu" && "${distro_ver}" == "22.04" ]]; then + expected_min="20260816ubuntu0.22.04.1" + if dpkg --compare-versions "${ca_pkg_ver}" ge "${expected_min}"; then + record_check "ca-certificates version" "INFO (OK)" "\`${ca_pkg_ver}\` >= \`${expected_min}\` (Ubuntu 22.04)" + else + record_check "ca-certificates version" "INFO (STALE)" "\`${ca_pkg_ver}\` < \`${expected_min}\` (Ubuntu 22.04)" + fi +elif [[ "${distro_id}" == "ubuntu" && "${distro_ver}" == "24.04" ]]; then + expected_min="20260816ubuntu0.24.04.1" + if dpkg --compare-versions "${ca_pkg_ver}" ge "${expected_min}"; then + record_check "ca-certificates version" "INFO (OK)" "\`${ca_pkg_ver}\` >= \`${expected_min}\` (Ubuntu 24.04)" + else + record_check "ca-certificates version" "INFO (STALE)" "\`${ca_pkg_ver}\` < \`${expected_min}\` (Ubuntu 24.04)" + fi +else + record_check "ca-certificates version" "INFO" "\`${ca_pkg_ver}\` on \`${distro_id} ${distro_ver}\`" +fi + +# 3. OS-store client (curl) rejects the test endpoint (only a successful TLS connection fails the job) +if [[ -z "${TEST_ENDPOINT}" ]]; then + record_check "curl rejects test endpoint" "SKIP" "Skipped via empty \`TRUST_PROBE_TEST_ENDPOINT\`" +elif ! command -v curl >/dev/null 2>&1; then + record_check "curl rejects test endpoint" "INCONCLUSIVE" "\`curl\` not found on PATH" +else + curl_rc=0 + curl_err="$(curl -sS --max-time 15 --cacert "${CA_BUNDLE}" -o /dev/null "${TEST_ENDPOINT}" 2>&1)" || curl_rc=$? + curl_err_oneline="$(tr '\n' ' ' <<< "${curl_err}" | sed 's/|/\\|/g')" + if (( curl_rc == 0 )); then + record_check "curl rejects test endpoint" "FAIL" "Connection to \`${TEST_ENDPOINT}\` succeeded (certificate was trusted)" + gating_failures=$((gating_failures + 1)) + elif (( curl_rc == 60 )); then + record_check "curl rejects test endpoint" "PASS" "Rejected with curl exit 60 (peer certificate cannot be authenticated)" + else + record_check "curl rejects test endpoint" "INCONCLUSIVE" "curl exited ${curl_rc} (network/DNS or non-verification error: ${curl_err_oneline})" + fi +fi + +append_summary "" + +if (( gating_failures > 0 )); then + echo "::error::${PROBE_LABEL}: ${gating_failures} gating check(s) failed" >&2 + exit 1 +fi diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index e1844384d54..63d1e806521 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -121,6 +121,10 @@ def test_ignored_paths_select_no_work(self) -> None: "diagram.svg", ".github/labeler.yml", ".github/ISSUE_TEMPLATE/bug.yml", + ".github/workflows/runner-trust-probe.yml", + "ci/tools/runner_trust_probe.sh", + "ci/tools/runner_trust_probe.ps1", + "ci/tools/tests/test_runner_trust_probe.py", ): with self.subTest(path=path): plan = plan_for(path) diff --git a/ci/tools/tests/test_runner_trust_probe.py b/ci/tools/tests/test_runner_trust_probe.py new file mode 100644 index 00000000000..571b307fb99 --- /dev/null +++ b/ci/tools/tests/test_runner_trust_probe.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the runner trust probe workflow and helper scripts.""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] +WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "runner-trust-probe.yml" +LINUX_SCRIPT = REPO_ROOT / "ci" / "tools" / "runner_trust_probe.sh" +WINDOWS_SCRIPT = REPO_ROOT / "ci" / "tools" / "runner_trust_probe.ps1" +EMSIGN_G3_SHA256 = "86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B" + + +def _split_pem_bundle(bundle_path: Path) -> list[tuple[str, str]]: + """Return (pem_text, colon_sha256_fingerprint) for each cert in a PEM bundle.""" + certs: list[tuple[str, str]] = [] + current: list[str] = [] + b64_lines: list[str] = [] + in_cert = False + for line in bundle_path.read_text(encoding="utf-8", errors="replace").splitlines(keepends=True): + if "-----BEGIN CERTIFICATE-----" in line: + in_cert = True + current = [line] + b64_lines = [] + elif "-----END CERTIFICATE-----" in line and in_cert: + current.append(line) + der = base64.b64decode("".join(b64_lines)) + digest = hashlib.sha256(der).hexdigest().upper() + fp = ":".join(digest[i : i + 2] for i in range(0, len(digest), 2)) + certs.append(("".join(current), fp)) + in_cert = False + elif in_cert: + current.append(line) + b64_lines.append(line.strip()) + return certs + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_runner_trust_probe_workflow_structure() -> None: + workflow = yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + if True in workflow: + workflow["on"] = workflow.pop(True) + + assert set(workflow["on"]) == {"schedule", "workflow_dispatch"} + assert workflow["permissions"] == {"contents": "read"} + assert set(workflow["jobs"]) == { + "compute-matrix", + "probe-linux-host", + "probe-linux-gpu", + "probe-windows", + } + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_linux_probe_passes_on_clean_bundle_and_fails_with_emsign_root(tmp_path: Path) -> None: + system_bundle = Path("/etc/ssl/certs/ca-certificates.crt") + if not system_bundle.is_file(): + pytest.skip("/etc/ssl/certs/ca-certificates.crt not present") + + certs = _split_pem_bundle(system_bundle) + clean_pems = [pem for pem, fp in certs if fp != EMSIGN_G3_SHA256][:5] + assert clean_pems, "Expected at least one non-emSign certificate in system bundle" + + clean_bundle = tmp_path / "clean.crt" + clean_bundle.write_text("".join(clean_pems), encoding="utf-8") + summary = tmp_path / "summary.md" + + env = dict(os.environ) + env["TRUST_PROBE_CA_BUNDLE"] = str(clean_bundle) + env["TRUST_PROBE_TEST_ENDPOINT"] = "" + env["GITHUB_STEP_SUMMARY"] = str(summary) + + res_clean = subprocess.run( # noqa: S603 + [str(LINUX_SCRIPT)], + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert res_clean.returncode == 0, res_clean.stdout + res_clean.stderr + assert "[PASS] Root anchor absent" in res_clean.stdout + assert "| Root anchor absent | **PASS** |" in summary.read_text(encoding="utf-8") + + # Test that presence of the fingerprint causes a gating failure. + # Reuse any certificate from the bundle and override TRUST_PROBE_TARGET_SHA256 + # (or use the actual emSign cert if present in the host bundle). + emsign_pems = [pem for pem, fp in certs if fp == EMSIGN_G3_SHA256] + if emsign_pems: + dirty_bundle = tmp_path / "dirty.crt" + dirty_bundle.write_text("".join(clean_pems) + emsign_pems[0], encoding="utf-8") + env_dirty = dict(env) + env_dirty["TRUST_PROBE_CA_BUNDLE"] = str(dirty_bundle) + else: + env_dirty = dict(env) + env_dirty["TRUST_PROBE_TARGET_SHA256"] = next(fp for _pem, fp in certs) + + res_dirty = subprocess.run( # noqa: S603 + [str(LINUX_SCRIPT)], + env=env_dirty, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert res_dirty.returncode == 1, res_dirty.stdout + res_dirty.stderr + assert "[FAIL] Root anchor absent" in res_dirty.stdout + + +@pytest.mark.agent_authored(model="gemini-4-argon") +def test_windows_probe_script_is_ascii_and_contains_fingerprint() -> None: + text = WINDOWS_SCRIPT.read_text(encoding="utf-8") + assert text.isascii() + assert EMSIGN_G3_SHA256 in text + assert "Cert:\\LocalMachine\\Root" in text + assert "Cert:\\LocalMachine\\Disallowed" in text From a7b5c91ffdad8b20d47bfc360e0d91551d04e680 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 10:32:36 -0700 Subject: [PATCH 02/10] TEMP: trigger runner-trust-probe on PR for pre-merge verification Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 22 +++++++++++++++++++++- ci/tools/tests/test_runner_trust_probe.py | 3 ++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 5ce40784c1d..4f7999db9c8 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -8,6 +8,22 @@ name: "CI: Runner trust probe" on: + # Temporary triggers for pre-merge PR verification (revert before merging): + # - pull_request exercises GitHub-hosted labels immediately on the PR. + # - push on pull-request/[0-9]+ exercises all labels (including self-hosted + # CPU and GPU runners) when synced by copy-pr-bot. + pull_request: + paths: + - ".github/workflows/runner-trust-probe.yml" + - "ci/tools/runner_trust_probe.sh" + - "ci/tools/runner_trust_probe.ps1" + push: + branches: + - "pull-request/[0-9]+" + paths: + - ".github/workflows/runner-trust-probe.yml" + - "ci/tools/runner_trust_probe.sh" + - "ci/tools/runner_trust_probe.ps1" schedule: # Weekly on Mondays at 06:23 UTC - cron: "23 6 * * 1" @@ -49,7 +65,11 @@ jobs: - name: Compute runner labels id: compute env: - LABEL_FILTER: ${{ inputs.label-filter || '' }} + # On fork pull_request events self-hosted runners do not pick up jobs; + # limit pull_request runs to GitHub-hosted labels, while push to + # pull-request/[0-9]+ (copy-pr-bot) and schedule/workflow_dispatch + # probe the full fleet. + LABEL_FILTER: ${{ inputs.label-filter || (github.event_name == 'pull_request' && '^(ubuntu-latest|windows-latest|windows-2022|windows-11-arm)$') || '' }} run: | matrix_json="$(yq -o json '.' ci/test-matrix.yml)" diff --git a/ci/tools/tests/test_runner_trust_probe.py b/ci/tools/tests/test_runner_trust_probe.py index 571b307fb99..6304fd439bd 100644 --- a/ci/tools/tests/test_runner_trust_probe.py +++ b/ci/tools/tests/test_runner_trust_probe.py @@ -51,7 +51,8 @@ def test_runner_trust_probe_workflow_structure() -> None: if True in workflow: workflow["on"] = workflow.pop(True) - assert set(workflow["on"]) == {"schedule", "workflow_dispatch"} + assert {"schedule", "workflow_dispatch"} <= set(workflow["on"]) + assert set(workflow["on"]) <= {"schedule", "workflow_dispatch", "pull_request", "push"} assert workflow["permissions"] == {"contents": "read"} assert set(workflow["jobs"]) == { "compute-matrix", From aa741b032721c284d36b27f88d319ac908e7ae21 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 10:47:43 -0700 Subject: [PATCH 03/10] Revert "TEMP: trigger runner-trust-probe on PR for pre-merge verification" This reverts commit a7b5c91ffdad8b20d47bfc360e0d91551d04e680. Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 22 +--------------------- ci/tools/tests/test_runner_trust_probe.py | 3 +-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 4f7999db9c8..5ce40784c1d 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -8,22 +8,6 @@ name: "CI: Runner trust probe" on: - # Temporary triggers for pre-merge PR verification (revert before merging): - # - pull_request exercises GitHub-hosted labels immediately on the PR. - # - push on pull-request/[0-9]+ exercises all labels (including self-hosted - # CPU and GPU runners) when synced by copy-pr-bot. - pull_request: - paths: - - ".github/workflows/runner-trust-probe.yml" - - "ci/tools/runner_trust_probe.sh" - - "ci/tools/runner_trust_probe.ps1" - push: - branches: - - "pull-request/[0-9]+" - paths: - - ".github/workflows/runner-trust-probe.yml" - - "ci/tools/runner_trust_probe.sh" - - "ci/tools/runner_trust_probe.ps1" schedule: # Weekly on Mondays at 06:23 UTC - cron: "23 6 * * 1" @@ -65,11 +49,7 @@ jobs: - name: Compute runner labels id: compute env: - # On fork pull_request events self-hosted runners do not pick up jobs; - # limit pull_request runs to GitHub-hosted labels, while push to - # pull-request/[0-9]+ (copy-pr-bot) and schedule/workflow_dispatch - # probe the full fleet. - LABEL_FILTER: ${{ inputs.label-filter || (github.event_name == 'pull_request' && '^(ubuntu-latest|windows-latest|windows-2022|windows-11-arm)$') || '' }} + LABEL_FILTER: ${{ inputs.label-filter || '' }} run: | matrix_json="$(yq -o json '.' ci/test-matrix.yml)" diff --git a/ci/tools/tests/test_runner_trust_probe.py b/ci/tools/tests/test_runner_trust_probe.py index 6304fd439bd..571b307fb99 100644 --- a/ci/tools/tests/test_runner_trust_probe.py +++ b/ci/tools/tests/test_runner_trust_probe.py @@ -51,8 +51,7 @@ def test_runner_trust_probe_workflow_structure() -> None: if True in workflow: workflow["on"] = workflow.pop(True) - assert {"schedule", "workflow_dispatch"} <= set(workflow["on"]) - assert set(workflow["on"]) <= {"schedule", "workflow_dispatch", "pull_request", "push"} + assert set(workflow["on"]) == {"schedule", "workflow_dispatch"} assert workflow["permissions"] == {"contents": "read"} assert set(workflow["jobs"]) == { "compute-matrix", From 49ba6491e41256e682a990e9cfa37899f16fa30f Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 12:15:49 -0700 Subject: [PATCH 04/10] CI: harden runner trust probe error classification and deduplicate Node script Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 179 +++------------------- ci/tools/compute_ci_plan.py | 2 +- ci/tools/runner_trust_probe.ps1 | 20 ++- ci/tools/runner_trust_probe.sh | 23 ++- ci/tools/runner_trust_probe_node.js | 61 ++++++++ ci/tools/tests/test_compute_ci_plan.py | 2 +- ci/tools/tests/test_runner_trust_probe.py | 126 --------------- 7 files changed, 111 insertions(+), 302 deletions(-) create mode 100644 ci/tools/runner_trust_probe_node.js delete mode 100644 ci/tools/tests/test_runner_trust_probe.py diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 5ce40784c1d..638d9a0fe4a 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -53,6 +53,8 @@ jobs: run: | matrix_json="$(yq -o json '.' ci/test-matrix.yml)" + # Hosted and self-hosted CPU labels are listed explicitly; GPU labels + # are derived from ci/test-matrix.yml. linux_host="$(jq -cn --arg filter "${LABEL_FILTER}" ' [ "ubuntu-latest", @@ -114,7 +116,6 @@ jobs: - name: Probe OS trust store id: os-probe - continue-on-error: ${{ matrix.label == 'ubuntu-latest' }} env: TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}" run: ./ci/tools/runner_trust_probe.sh @@ -122,7 +123,6 @@ jobs: - name: Probe OS trust store after ca-certificates upgrade (ubuntu-latest) id: os-probe-upgraded if: ${{ always() && matrix.label == 'ubuntu-latest' }} - continue-on-error: true env: TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)" run: | @@ -149,71 +149,26 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const https = require('https'); - const version = process.version; - const execPath = process.execPath; - const nodeOptions = process.env.NODE_OPTIONS || ''; - const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; - const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; - - const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); - const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; - const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); - - const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; - const probeEndpoint = () => new Promise((resolve) => { - const req = https.get(endpoint, { timeout: 15000 }, (res) => { - res.resume(); - resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); - }); - req.on('timeout', () => { - req.destroy(new Error('Request timed out')); - }); - req.on('error', (err) => { - const code = err.code || ''; - const msg = `${code} ${err.message || ''}`.trim(); - if (/CERT|UNABLE_TO_VERIFY|SELF_SIGNED|DEPTH_ZERO|SSL|TLS/i.test(msg)) { - resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); - } else { - resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); - } - }); - }); - - const tlsResult = await probeEndpoint(); - const rows = [ - ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require \`>= v24.22.1\` on Node 24; \`execPath=${execPath}\`)`], - ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], - ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], - ]; - - await core.summary - .addHeading('Node runtime trust store', 3) - .addTable([ - [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], - ...rows, - ]) - .write(); - - if (!versionOk || tlsResult.status === 'FAIL') { - core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); - } + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core }); - name: Check probe outcomes if: ${{ always() }} run: | + failed=0 if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then echo "::error::Initial OS trust store probe failed" >&2 - exit 1 + failed=1 fi if [[ "${{ matrix.label }}" == "ubuntu-latest" && "${{ steps.os-probe-upgraded.outcome }}" != "success" ]]; then echo "::error::Post-upgrade OS trust store probe failed on ubuntu-latest" >&2 - exit 1 + failed=1 fi if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then echo "::error::Node runtime trust store probe failed" >&2 - exit 1 + failed=1 fi + exit "${failed}" probe-linux-gpu: name: Probe (${{ matrix.label }}) @@ -250,7 +205,6 @@ jobs: - name: Probe container OS trust store id: os-probe - continue-on-error: true env: TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)" run: ./ci/tools/runner_trust_probe.sh @@ -274,67 +228,22 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const https = require('https'); - const version = process.version; - const execPath = process.execPath; - const nodeOptions = process.env.NODE_OPTIONS || ''; - const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; - const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; - - const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); - const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; - const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); - - const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; - const probeEndpoint = () => new Promise((resolve) => { - const req = https.get(endpoint, { timeout: 15000 }, (res) => { - res.resume(); - resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); - }); - req.on('timeout', () => { - req.destroy(new Error('Request timed out')); - }); - req.on('error', (err) => { - const code = err.code || ''; - const msg = `${code} ${err.message || ''}`.trim(); - if (/CERT|UNABLE_TO_VERIFY|SELF_SIGNED|DEPTH_ZERO|SSL|TLS/i.test(msg)) { - resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); - } else { - resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); - } - }); - }); - - const tlsResult = await probeEndpoint(); - const rows = [ - ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require \`>= v24.22.1\` on Node 24; \`execPath=${execPath}\`)`], - ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], - ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], - ]; - - await core.summary - .addHeading('Node runtime trust store', 3) - .addTable([ - [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], - ...rows, - ]) - .write(); - - if (!versionOk || tlsResult.status === 'FAIL') { - core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); - } + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core }); - name: Check probe outcomes if: ${{ always() }} run: | + failed=0 if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then echo "::error::Container OS trust store probe failed" >&2 - exit 1 + failed=1 fi if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then echo "::error::Node runtime trust store probe failed" >&2 - exit 1 + failed=1 fi + exit "${failed}" probe-windows: name: Probe (${{ matrix.label }}) @@ -355,7 +264,6 @@ jobs: - name: Probe Windows OS trust store id: os-probe - continue-on-error: true shell: pwsh run: ./ci/tools/runner_trust_probe.ps1 @@ -378,64 +286,19 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const https = require('https'); - const version = process.version; - const execPath = process.execPath; - const nodeOptions = process.env.NODE_OPTIONS || ''; - const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; - const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; - - const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); - const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; - const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); - - const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; - const probeEndpoint = () => new Promise((resolve) => { - const req = https.get(endpoint, { timeout: 15000 }, (res) => { - res.resume(); - resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); - }); - req.on('timeout', () => { - req.destroy(new Error('Request timed out')); - }); - req.on('error', (err) => { - const code = err.code || ''; - const msg = `${code} ${err.message || ''}`.trim(); - if (/CERT|UNABLE_TO_VERIFY|SELF_SIGNED|DEPTH_ZERO|SSL|TLS/i.test(msg)) { - resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); - } else { - resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); - } - }); - }); - - const tlsResult = await probeEndpoint(); - const rows = [ - ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require \`>= v24.22.1\` on Node 24; \`execPath=${execPath}\`)`], - ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], - ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], - ]; - - await core.summary - .addHeading('Node runtime trust store', 3) - .addTable([ - [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], - ...rows, - ]) - .write(); - - if (!versionOk || tlsResult.status === 'FAIL') { - core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); - } + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core }); - name: Check probe outcomes if: ${{ always() }} run: | + failed=0 if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then echo "::error::Windows OS trust store probe failed" >&2 - exit 1 + failed=1 fi if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then echo "::error::Node runtime trust store probe failed" >&2 - exit 1 + failed=1 fi + exit "${failed}" diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py index 9042418a167..51493044569 100644 --- a/ci/tools/compute_ci_plan.py +++ b/ci/tools/compute_ci_plan.py @@ -44,7 +44,7 @@ "LICENSE", "ci/tools/runner_trust_probe.ps1", "ci/tools/runner_trust_probe.sh", - "ci/tools/tests/test_runner_trust_probe.py", + "ci/tools/runner_trust_probe_node.js", "context7.json", "greptile.json", "ruff.toml", diff --git a/ci/tools/runner_trust_probe.ps1 b/ci/tools/runner_trust_probe.ps1 index 23a522e27a1..31d8845476e 100644 --- a/ci/tools/runner_trust_probe.ps1 +++ b/ci/tools/runner_trust_probe.ps1 @@ -84,11 +84,23 @@ if ([string]::IsNullOrEmpty($TestEndpoint)) { Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded (certificate was trusted)" $GatingFailures++ } catch { - $msg = ($_.Exception.ToString() -replace '[\r\n]+', ' ' -replace '\|', '\|') - if ($msg -match 'AuthenticationException|TrustFailure|SSL|TLS|certificate|Cert| remote party') { - Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with TLS/certificate error" + $certRejected = $false + $ex = $_.Exception + while ($null -ne $ex) { + if ( + ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'remote certificate|certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or + ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure) + ) { + $certRejected = $true + break + } + $ex = $ex.InnerException + } + $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|') + if ($certRejected) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with certificate validation error ($cleanMsg)" } else { - Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-TLS or network error: $($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|')" + Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)" } } finally { $ProgressPreference = $progTmp diff --git a/ci/tools/runner_trust_probe.sh b/ci/tools/runner_trust_probe.sh index 5db50f23d92..870526623f0 100755 --- a/ci/tools/runner_trust_probe.sh +++ b/ci/tools/runner_trust_probe.sh @@ -13,7 +13,7 @@ set -euo pipefail -EMSIGN_G3_SHA256="${TRUST_PROBE_TARGET_SHA256:-86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B}" +EMSIGN_G3_SHA256="86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B" CA_BUNDLE="${TRUST_PROBE_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" TEST_ENDPOINT="${TRUST_PROBE_TEST_ENDPOINT:-https://dv-g3b-emudhra.we-have-a-problem.com/}" PROBE_LABEL="${TRUST_PROBE_LABEL:-Linux OS trust store}" @@ -94,21 +94,20 @@ if [[ -f /etc/os-release ]]; then distro_ver="${VERSION_ID:-unknown}" fi +case "${distro_id}:${distro_ver}" in + ubuntu:22.04) expected_min="20260816ubuntu0.22.04.1" ;; + ubuntu:24.04) expected_min="20260816ubuntu0.24.04.1" ;; + ubuntu:26.04) expected_min="20260816ubuntu0.26.04.1" ;; + *) expected_min="" ;; +esac + if [[ "${ca_pkg_ver}" == "not-installed" ]]; then record_check "ca-certificates version" "INFO" "Package not installed (distro: \`${distro_id} ${distro_ver}\`)" -elif [[ "${distro_id}" == "ubuntu" && "${distro_ver}" == "22.04" ]]; then - expected_min="20260816ubuntu0.22.04.1" - if dpkg --compare-versions "${ca_pkg_ver}" ge "${expected_min}"; then - record_check "ca-certificates version" "INFO (OK)" "\`${ca_pkg_ver}\` >= \`${expected_min}\` (Ubuntu 22.04)" - else - record_check "ca-certificates version" "INFO (STALE)" "\`${ca_pkg_ver}\` < \`${expected_min}\` (Ubuntu 22.04)" - fi -elif [[ "${distro_id}" == "ubuntu" && "${distro_ver}" == "24.04" ]]; then - expected_min="20260816ubuntu0.24.04.1" +elif [[ -n "${expected_min}" ]]; then if dpkg --compare-versions "${ca_pkg_ver}" ge "${expected_min}"; then - record_check "ca-certificates version" "INFO (OK)" "\`${ca_pkg_ver}\` >= \`${expected_min}\` (Ubuntu 24.04)" + record_check "ca-certificates version" "INFO (OK)" "\`${ca_pkg_ver}\` >= \`${expected_min}\` (Ubuntu ${distro_ver})" else - record_check "ca-certificates version" "INFO (STALE)" "\`${ca_pkg_ver}\` < \`${expected_min}\` (Ubuntu 24.04)" + record_check "ca-certificates version" "INFO (STALE)" "\`${ca_pkg_ver}\` < \`${expected_min}\` (Ubuntu ${distro_ver})" fi else record_check "ca-certificates version" "INFO" "\`${ca_pkg_ver}\` on \`${distro_id} ${distro_ver}\`" diff --git a/ci/tools/runner_trust_probe_node.js b/ci/tools/runner_trust_probe_node.js new file mode 100644 index 00000000000..b7d15136746 --- /dev/null +++ b/ci/tools/runner_trust_probe_node.js @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// runner_trust_probe_node.js -- probe the runner agent's bundled Node.js +// version and certificate store for the eMudhra/emSign CA incident. + +'use strict'; + +const https = require('https'); + +module.exports = async function runNodeTrustProbe({ core }) { + const version = process.version; + const execPath = process.execPath; + const nodeOptions = process.env.NODE_OPTIONS || ''; + const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; + const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; + + const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); + const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; + const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); + + const endpoint = process.env.TRUST_PROBE_TEST_ENDPOINT || 'https://dv-g3b-emudhra.we-have-a-problem.com/'; + const probeEndpoint = () => new Promise((resolve) => { + const req = https.get(endpoint, { timeout: 15000 }, (res) => { + res.resume(); + resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); + }); + req.on('timeout', () => { + req.destroy(new Error('Request timed out')); + }); + req.on('error', (err) => { + const code = err.code || ''; + const msg = `${code} ${err.message || ''}`.trim(); + if (/^(CERT_|UNABLE_TO_|SELF_SIGNED_|DEPTH_ZERO_|ERR_TLS_CERT_)/.test(code)) { + resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); + } else { + resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); + } + }); + }); + + const tlsResult = await probeEndpoint(); + const rows = [ + ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require Node 24 \`>= v24.22.1\`; \`execPath=${execPath}\`)`], + ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], + ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], + ]; + + await core.summary + .addHeading('Node runtime trust store', 3) + .addTable([ + [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], + ...rows, + ]) + .write(); + + if (!versionOk || tlsResult.status === 'FAIL') { + core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); + } +}; diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index 63d1e806521..b87326420ee 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -124,7 +124,7 @@ def test_ignored_paths_select_no_work(self) -> None: ".github/workflows/runner-trust-probe.yml", "ci/tools/runner_trust_probe.sh", "ci/tools/runner_trust_probe.ps1", - "ci/tools/tests/test_runner_trust_probe.py", + "ci/tools/runner_trust_probe_node.js", ): with self.subTest(path=path): plan = plan_for(path) diff --git a/ci/tools/tests/test_runner_trust_probe.py b/ci/tools/tests/test_runner_trust_probe.py deleted file mode 100644 index 571b307fb99..00000000000 --- a/ci/tools/tests/test_runner_trust_probe.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for the runner trust probe workflow and helper scripts.""" - -from __future__ import annotations - -import base64 -import hashlib -import os -import subprocess -from pathlib import Path - -import pytest -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[3] -WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "runner-trust-probe.yml" -LINUX_SCRIPT = REPO_ROOT / "ci" / "tools" / "runner_trust_probe.sh" -WINDOWS_SCRIPT = REPO_ROOT / "ci" / "tools" / "runner_trust_probe.ps1" -EMSIGN_G3_SHA256 = "86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B" - - -def _split_pem_bundle(bundle_path: Path) -> list[tuple[str, str]]: - """Return (pem_text, colon_sha256_fingerprint) for each cert in a PEM bundle.""" - certs: list[tuple[str, str]] = [] - current: list[str] = [] - b64_lines: list[str] = [] - in_cert = False - for line in bundle_path.read_text(encoding="utf-8", errors="replace").splitlines(keepends=True): - if "-----BEGIN CERTIFICATE-----" in line: - in_cert = True - current = [line] - b64_lines = [] - elif "-----END CERTIFICATE-----" in line and in_cert: - current.append(line) - der = base64.b64decode("".join(b64_lines)) - digest = hashlib.sha256(der).hexdigest().upper() - fp = ":".join(digest[i : i + 2] for i in range(0, len(digest), 2)) - certs.append(("".join(current), fp)) - in_cert = False - elif in_cert: - current.append(line) - b64_lines.append(line.strip()) - return certs - - -@pytest.mark.agent_authored(model="gemini-4-argon") -def test_runner_trust_probe_workflow_structure() -> None: - workflow = yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) - if True in workflow: - workflow["on"] = workflow.pop(True) - - assert set(workflow["on"]) == {"schedule", "workflow_dispatch"} - assert workflow["permissions"] == {"contents": "read"} - assert set(workflow["jobs"]) == { - "compute-matrix", - "probe-linux-host", - "probe-linux-gpu", - "probe-windows", - } - - -@pytest.mark.agent_authored(model="gemini-4-argon") -def test_linux_probe_passes_on_clean_bundle_and_fails_with_emsign_root(tmp_path: Path) -> None: - system_bundle = Path("/etc/ssl/certs/ca-certificates.crt") - if not system_bundle.is_file(): - pytest.skip("/etc/ssl/certs/ca-certificates.crt not present") - - certs = _split_pem_bundle(system_bundle) - clean_pems = [pem for pem, fp in certs if fp != EMSIGN_G3_SHA256][:5] - assert clean_pems, "Expected at least one non-emSign certificate in system bundle" - - clean_bundle = tmp_path / "clean.crt" - clean_bundle.write_text("".join(clean_pems), encoding="utf-8") - summary = tmp_path / "summary.md" - - env = dict(os.environ) - env["TRUST_PROBE_CA_BUNDLE"] = str(clean_bundle) - env["TRUST_PROBE_TEST_ENDPOINT"] = "" - env["GITHUB_STEP_SUMMARY"] = str(summary) - - res_clean = subprocess.run( # noqa: S603 - [str(LINUX_SCRIPT)], - env=env, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - assert res_clean.returncode == 0, res_clean.stdout + res_clean.stderr - assert "[PASS] Root anchor absent" in res_clean.stdout - assert "| Root anchor absent | **PASS** |" in summary.read_text(encoding="utf-8") - - # Test that presence of the fingerprint causes a gating failure. - # Reuse any certificate from the bundle and override TRUST_PROBE_TARGET_SHA256 - # (or use the actual emSign cert if present in the host bundle). - emsign_pems = [pem for pem, fp in certs if fp == EMSIGN_G3_SHA256] - if emsign_pems: - dirty_bundle = tmp_path / "dirty.crt" - dirty_bundle.write_text("".join(clean_pems) + emsign_pems[0], encoding="utf-8") - env_dirty = dict(env) - env_dirty["TRUST_PROBE_CA_BUNDLE"] = str(dirty_bundle) - else: - env_dirty = dict(env) - env_dirty["TRUST_PROBE_TARGET_SHA256"] = next(fp for _pem, fp in certs) - - res_dirty = subprocess.run( # noqa: S603 - [str(LINUX_SCRIPT)], - env=env_dirty, - capture_output=True, - text=True, - timeout=30, - check=False, - ) - assert res_dirty.returncode == 1, res_dirty.stdout + res_dirty.stderr - assert "[FAIL] Root anchor absent" in res_dirty.stdout - - -@pytest.mark.agent_authored(model="gemini-4-argon") -def test_windows_probe_script_is_ascii_and_contains_fingerprint() -> None: - text = WINDOWS_SCRIPT.read_text(encoding="utf-8") - assert text.isascii() - assert EMSIGN_G3_SHA256 in text - assert "Cert:\\LocalMachine\\Root" in text - assert "Cert:\\LocalMachine\\Disallowed" in text From 171b26b7cdb89d5c50a9d1c20436d102b487bce4 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 13:23:13 -0700 Subject: [PATCH 05/10] CI: tighten TLS certificate chain vs. hostname error classification Signed-off-by: Ralf Juengling --- ci/tools/runner_trust_probe.ps1 | 9 ++++++++- ci/tools/runner_trust_probe_node.js | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ci/tools/runner_trust_probe.ps1 b/ci/tools/runner_trust_probe.ps1 index 31d8845476e..0288c913099 100644 --- a/ci/tools/runner_trust_probe.ps1 +++ b/ci/tools/runner_trust_probe.ps1 @@ -85,20 +85,27 @@ if ([string]::IsNullOrEmpty($TestEndpoint)) { $GatingFailures++ } catch { $certRejected = $false + $hostnameMismatchOnly = $false $ex = $_.Exception while ($null -ne $ex) { if ( - ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'remote certificate|certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or + ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure) ) { $certRejected = $true break } + if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') { + $hostnameMismatchOnly = $true + } $ex = $ex.InnerException } $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|') if ($certRejected) { Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with certificate validation error ($cleanMsg)" + } elseif ($hostnameMismatchOnly) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)" + $GatingFailures++ } else { Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)" } diff --git a/ci/tools/runner_trust_probe_node.js b/ci/tools/runner_trust_probe_node.js index b7d15136746..5afa460f3d4 100644 --- a/ci/tools/runner_trust_probe_node.js +++ b/ci/tools/runner_trust_probe_node.js @@ -32,7 +32,9 @@ module.exports = async function runNodeTrustProbe({ core }) { req.on('error', (err) => { const code = err.code || ''; const msg = `${code} ${err.message || ''}`.trim(); - if (/^(CERT_|UNABLE_TO_|SELF_SIGNED_|DEPTH_ZERO_|ERR_TLS_CERT_)/.test(code)) { + if (code === 'ERR_TLS_CERT_ALTNAME_INVALID') { + resolve({ status: 'FAIL', detail: `Certificate chain was trusted; failed only hostname check (${msg})` }); + } else if (/^(CERT_|UNABLE_TO_|SELF_SIGNED_|DEPTH_ZERO_)/.test(code)) { resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); } else { resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); From 1070160643551a6932810f30bd4bfb70dcefa486 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 13:23:44 -0700 Subject: [PATCH 06/10] TEMP: trigger runner-trust-probe on PR for pre-merge verification Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 638d9a0fe4a..9bc2610e454 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -8,6 +8,24 @@ name: "CI: Runner trust probe" on: + # Temporary triggers for pre-merge PR verification (revert before merging): + # - pull_request exercises GitHub-hosted labels immediately on the PR. + # - push on pull-request/[0-9]+ exercises all labels (including self-hosted + # CPU and GPU runners) when synced by copy-pr-bot. + pull_request: + paths: + - ".github/workflows/runner-trust-probe.yml" + - "ci/tools/runner_trust_probe.sh" + - "ci/tools/runner_trust_probe.ps1" + - "ci/tools/runner_trust_probe_node.js" + push: + branches: + - "pull-request/[0-9]+" + paths: + - ".github/workflows/runner-trust-probe.yml" + - "ci/tools/runner_trust_probe.sh" + - "ci/tools/runner_trust_probe.ps1" + - "ci/tools/runner_trust_probe_node.js" schedule: # Weekly on Mondays at 06:23 UTC - cron: "23 6 * * 1" @@ -49,7 +67,11 @@ jobs: - name: Compute runner labels id: compute env: - LABEL_FILTER: ${{ inputs.label-filter || '' }} + # On fork pull_request events self-hosted runners do not pick up jobs; + # limit pull_request runs to GitHub-hosted labels, while push to + # pull-request/[0-9]+ (copy-pr-bot) and schedule/workflow_dispatch + # probe the full fleet. + LABEL_FILTER: ${{ inputs.label-filter || (github.event_name == 'pull_request' && '^(ubuntu-latest|windows-latest|windows-2022|windows-11-arm)$') || '' }} run: | matrix_json="$(yq -o json '.' ci/test-matrix.yml)" From 711671b766f280195c6f323aa1dadf780bad6d86 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 13:27:13 -0700 Subject: [PATCH 07/10] Revert "TEMP: trigger runner-trust-probe on PR for pre-merge verification" This reverts commit 1070160643551a6932810f30bd4bfb70dcefa486. Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 9bc2610e454..638d9a0fe4a 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -8,24 +8,6 @@ name: "CI: Runner trust probe" on: - # Temporary triggers for pre-merge PR verification (revert before merging): - # - pull_request exercises GitHub-hosted labels immediately on the PR. - # - push on pull-request/[0-9]+ exercises all labels (including self-hosted - # CPU and GPU runners) when synced by copy-pr-bot. - pull_request: - paths: - - ".github/workflows/runner-trust-probe.yml" - - "ci/tools/runner_trust_probe.sh" - - "ci/tools/runner_trust_probe.ps1" - - "ci/tools/runner_trust_probe_node.js" - push: - branches: - - "pull-request/[0-9]+" - paths: - - ".github/workflows/runner-trust-probe.yml" - - "ci/tools/runner_trust_probe.sh" - - "ci/tools/runner_trust_probe.ps1" - - "ci/tools/runner_trust_probe_node.js" schedule: # Weekly on Mondays at 06:23 UTC - cron: "23 6 * * 1" @@ -67,11 +49,7 @@ jobs: - name: Compute runner labels id: compute env: - # On fork pull_request events self-hosted runners do not pick up jobs; - # limit pull_request runs to GitHub-hosted labels, while push to - # pull-request/[0-9]+ (copy-pr-bot) and schedule/workflow_dispatch - # probe the full fleet. - LABEL_FILTER: ${{ inputs.label-filter || (github.event_name == 'pull_request' && '^(ubuntu-latest|windows-latest|windows-2022|windows-11-arm)$') || '' }} + LABEL_FILTER: ${{ inputs.label-filter || '' }} run: | matrix_json="$(yq -o json '.' ci/test-matrix.yml)" From 756c2d6346874c1e27777eac3d9c2c986c897000 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 13:42:51 -0700 Subject: [PATCH 08/10] CI: distinguish curl hostname mismatch from CA rejection in trust probe Signed-off-by: Ralf Juengling --- ci/tools/runner_trust_probe.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/tools/runner_trust_probe.sh b/ci/tools/runner_trust_probe.sh index 870526623f0..126f1c4cc59 100755 --- a/ci/tools/runner_trust_probe.sh +++ b/ci/tools/runner_trust_probe.sh @@ -126,7 +126,12 @@ else record_check "curl rejects test endpoint" "FAIL" "Connection to \`${TEST_ENDPOINT}\` succeeded (certificate was trusted)" gating_failures=$((gating_failures + 1)) elif (( curl_rc == 60 )); then - record_check "curl rejects test endpoint" "PASS" "Rejected with curl exit 60 (peer certificate cannot be authenticated)" + if grep -qiE 'no alternative certificate subject name|subject alternative name|does not match' <<< "${curl_err}"; then + record_check "curl rejects test endpoint" "FAIL" "Certificate chain was trusted; failed only hostname check (${curl_err_oneline})" + gating_failures=$((gating_failures + 1)) + else + record_check "curl rejects test endpoint" "PASS" "Rejected with curl exit 60 (peer certificate cannot be authenticated)" + fi else record_check "curl rejects test endpoint" "INCONCLUSIVE" "curl exited ${curl_rc} (network/DNS or non-verification error: ${curl_err_oneline})" fi From 2d119d7a6537c139a85b52b36f3568c124d00f61 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 14:17:55 -0700 Subject: [PATCH 09/10] CI: simplify runner trust probe workflow and scripts Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 200 +++++------------------ ci/tools/runner_trust_probe.ps1 | 119 +++++--------- ci/tools/runner_trust_probe.sh | 87 +++------- ci/tools/runner_trust_probe_node.js | 12 +- 4 files changed, 112 insertions(+), 306 deletions(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 638d9a0fe4a..7a974d27ad9 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -33,17 +33,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: - linux_host: ${{ steps.compute.outputs.linux_host }} + host: ${{ steps.compute.outputs.host }} linux_gpu: ${{ steps.compute.outputs.linux_gpu }} - windows: ${{ steps.compute.outputs.windows }} - has_linux_host: ${{ steps.compute.outputs.has_linux_host }} - has_linux_gpu: ${{ steps.compute.outputs.has_linux_gpu }} - has_windows: ${{ steps.compute.outputs.has_windows }} steps: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 1 persist-credentials: false - name: Compute runner labels @@ -51,77 +46,58 @@ jobs: env: LABEL_FILTER: ${{ inputs.label-filter || '' }} run: | - matrix_json="$(yq -o json '.' ci/test-matrix.yml)" - # Hosted and self-hosted CPU labels are listed explicitly; GPU labels # are derived from ci/test-matrix.yml. - linux_host="$(jq -cn --arg filter "${LABEL_FILTER}" ' - [ - "ubuntu-latest", - "linux-amd64-cpu8", - "linux-arm64-cpu8", - "linux-amd64-cpu4" - ] - | unique - | if $filter != "" then map(select(test($filter))) else . end - ')" - - linux_gpu="$(jq -cn --argjson m "${matrix_json}" --arg filter "${LABEL_FILTER}" ' - [ - ($m.linux["pull-request"][]?, $m.linux["nightly"][]?) - | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" - ] - | unique - | if $filter != "" then map(select(test($filter))) else . end - ')" - - windows="$(jq -cn --argjson m "${matrix_json}" --arg filter "${LABEL_FILTER}" ' - ( - [ - "windows-latest", - "windows-2022", - "windows-11-arm" - ] + [ - ($m.windows["pull-request"][]?, $m.windows["nightly"][]?) - | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" + labels_json="$(yq -o json '.' ci/test-matrix.yml | jq -c --arg filter "${LABEL_FILTER}" ' + { + host: ( + [ + "ubuntu-latest", + "linux-amd64-cpu8", + "linux-arm64-cpu8", + "linux-amd64-cpu4", + "windows-latest", + "windows-2022", + "windows-11-arm" + ] + [ + (.windows["pull-request"][]?, .windows["nightly"][]?) + | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" + ] + ), + linux_gpu: [ + (.linux["pull-request"][]?, .linux["nightly"][]?) + | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" ] - ) - | unique - | if $filter != "" then map(select(test($filter))) else . end + } + | map_values(unique | if $filter != "" then map(select(test($filter))) else . end) ')" - echo "linux_host=${linux_host}" | tee -a "${GITHUB_OUTPUT}" - echo "linux_gpu=${linux_gpu}" | tee -a "${GITHUB_OUTPUT}" - echo "windows=${windows}" | tee -a "${GITHUB_OUTPUT}" - echo "has_linux_host=$(jq 'length > 0' <<< "${linux_host}")" | tee -a "${GITHUB_OUTPUT}" - echo "has_linux_gpu=$(jq 'length > 0' <<< "${linux_gpu}")" | tee -a "${GITHUB_OUTPUT}" - echo "has_windows=$(jq 'length > 0' <<< "${windows}")" | tee -a "${GITHUB_OUTPUT}" + echo "host=$(jq -c '.host' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}" + echo "linux_gpu=$(jq -c '.linux_gpu' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}" - probe-linux-host: + probe-host: name: Probe (${{ matrix.label }}) needs: compute-matrix - if: ${{ github.repository_owner == 'nvidia' && needs.compute-matrix.outputs.has_linux_host == 'true' }} + if: ${{ needs.compute-matrix.outputs.host != '[]' }} strategy: fail-fast: false matrix: - label: ${{ fromJSON(needs.compute-matrix.outputs.linux_host) }} + label: ${{ fromJSON(needs.compute-matrix.outputs.host) }} runs-on: ${{ matrix.label }} timeout-minutes: 15 steps: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 1 persist-credentials: false - - name: Probe OS trust store - id: os-probe + - name: Probe Linux OS trust store + if: ${{ runner.os != 'Windows' }} env: TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}" run: ./ci/tools/runner_trust_probe.sh - - name: Probe OS trust store after ca-certificates upgrade (ubuntu-latest) - id: os-probe-upgraded + - name: Probe Linux OS trust store after ca-certificates upgrade (ubuntu-latest) if: ${{ always() && matrix.label == 'ubuntu-latest' }} env: TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)" @@ -130,21 +106,21 @@ jobs: sudo apt-get install --only-upgrade -y ca-certificates ./ci/tools/runner_trust_probe.sh + - name: Probe Windows OS trust store + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: ./ci/tools/runner_trust_probe.ps1 + - name: Probe Node 20 action runtime (informational) if: ${{ always() }} continue-on-error: true uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) with: script: | - const msg = `Node 20 action runtime resolved to \`${process.version}\` (\`${process.execPath}\`)`; - core.info(msg); - await core.summary - .addHeading('Node 20 action runtime (informational)', 3) - .addRaw(msg + '\n\n') - .write(); + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false }); - name: Probe bundled Node runtime and trust store - id: node-probe if: ${{ always() }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -152,28 +128,10 @@ jobs: const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); await probe({ core }); - - name: Check probe outcomes - if: ${{ always() }} - run: | - failed=0 - if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then - echo "::error::Initial OS trust store probe failed" >&2 - failed=1 - fi - if [[ "${{ matrix.label }}" == "ubuntu-latest" && "${{ steps.os-probe-upgraded.outcome }}" != "success" ]]; then - echo "::error::Post-upgrade OS trust store probe failed on ubuntu-latest" >&2 - failed=1 - fi - if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then - echo "::error::Node runtime trust store probe failed" >&2 - failed=1 - fi - exit "${failed}" - probe-linux-gpu: name: Probe (${{ matrix.label }}) needs: compute-matrix - if: ${{ github.repository_owner == 'nvidia' && needs.compute-matrix.outputs.has_linux_gpu == 'true' }} + if: ${{ needs.compute-matrix.outputs.linux_gpu != '[]' }} strategy: fail-fast: false matrix: @@ -189,7 +147,6 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 1 persist-credentials: false - name: Setup proxy cache @@ -198,13 +155,11 @@ jobs: - name: Install container CA certificates and probe tools uses: ./.github/actions/install_unix_deps - continue-on-error: false with: dependencies: "ca-certificates curl openssl" dependent_exes: "curl openssl" - name: Probe container OS trust store - id: os-probe env: TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)" run: ./ci/tools/runner_trust_probe.sh @@ -213,92 +168,15 @@ jobs: if: ${{ always() }} continue-on-error: true uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) - with: - script: | - const msg = `Node 20 action runtime resolved to \`${process.version}\` (\`${process.execPath}\`)`; - core.info(msg); - await core.summary - .addHeading('Node 20 action runtime (informational)', 3) - .addRaw(msg + '\n\n') - .write(); - - - name: Probe bundled Node runtime and trust store - id: node-probe - if: ${{ always() }} - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); - await probe({ core }); - - - name: Check probe outcomes - if: ${{ always() }} - run: | - failed=0 - if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then - echo "::error::Container OS trust store probe failed" >&2 - failed=1 - fi - if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then - echo "::error::Node runtime trust store probe failed" >&2 - failed=1 - fi - exit "${failed}" - - probe-windows: - name: Probe (${{ matrix.label }}) - needs: compute-matrix - if: ${{ github.repository_owner == 'nvidia' && needs.compute-matrix.outputs.has_windows == 'true' }} - strategy: - fail-fast: false - matrix: - label: ${{ fromJSON(needs.compute-matrix.outputs.windows) }} - runs-on: ${{ matrix.label }} - timeout-minutes: 15 - steps: - - name: Checkout ${{ github.event.repository.name }} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 1 - persist-credentials: false - - - name: Probe Windows OS trust store - id: os-probe - shell: pwsh - run: ./ci/tools/runner_trust_probe.ps1 - - - name: Probe Node 20 action runtime (informational) - if: ${{ always() }} - continue-on-error: true - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) - with: - script: | - const msg = `Node 20 action runtime resolved to \`${process.version}\` (\`${process.execPath}\`)`; - core.info(msg); - await core.summary - .addHeading('Node 20 action runtime (informational)', 3) - .addRaw(msg + '\n\n') - .write(); + await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false }); - name: Probe bundled Node runtime and trust store - id: node-probe if: ${{ always() }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); await probe({ core }); - - - name: Check probe outcomes - if: ${{ always() }} - run: | - failed=0 - if [[ "${{ steps.os-probe.outcome }}" != "success" ]]; then - echo "::error::Windows OS trust store probe failed" >&2 - failed=1 - fi - if [[ "${{ steps.node-probe.outcome }}" != "success" ]]; then - echo "::error::Node runtime trust store probe failed" >&2 - failed=1 - fi - exit "${failed}" diff --git a/ci/tools/runner_trust_probe.ps1 b/ci/tools/runner_trust_probe.ps1 index 0288c913099..8bf103d6d6f 100644 --- a/ci/tools/runner_trust_probe.ps1 +++ b/ci/tools/runner_trust_probe.ps1 @@ -4,17 +4,12 @@ # # runner_trust_probe.ps1 -- probe the Windows certificate store on a CI runner # for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3). -# -# Optional environment overrides for local testing: -# TRUST_PROBE_TEST_ENDPOINT HTTPS endpoint to test (default: https://dv-g3b-emudhra.we-have-a-problem.com/) -# TRUST_PROBE_LABEL Heading label for GITHUB_STEP_SUMMARY $ErrorActionPreference = 'Stop' -$EmSignG3Sha256 = '86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B' -$EmSignG3Hex = $EmSignG3Sha256.Replace(':', '') -$TestEndpoint = if ($null -ne $env:TRUST_PROBE_TEST_ENDPOINT) { $env:TRUST_PROBE_TEST_ENDPOINT } else { 'https://dv-g3b-emudhra.we-have-a-problem.com/' } -$ProbeLabel = if ($env:TRUST_PROBE_LABEL) { $env:TRUST_PROBE_LABEL } else { 'Windows OS trust store' } +$EmSignG3Hex = '86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B' +$TestEndpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/' +$ProbeLabel = 'Windows OS trust store' $SummaryFile = $env:GITHUB_STEP_SUMMARY function Append-Summary([string]$Line) { @@ -36,82 +31,54 @@ Append-Summary "| Check | Status | Details |" Append-Summary "| --- | --- | --- |" # 1. Root anchor not trusted in Cert:\LocalMachine\Root (or present in Disallowed) (gating) -$sha256 = [System.Security.Cryptography.SHA256]::Create() +$rootCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Root' -ErrorAction Stop) +$inRoot = [bool]($rootCerts | Where-Object { $_.GetCertHashString('SHA256') -eq $EmSignG3Hex }) +$inDisallowed = [bool](Get-ChildItem -Path 'Cert:\LocalMachine\Disallowed' -ErrorAction SilentlyContinue | + Where-Object { $_.GetCertHashString('SHA256') -eq $EmSignG3Hex }) + +$storeSummary = "LocalMachine\Root=$inRoot ($($rootCerts.Count) certs), LocalMachine\Disallowed=$inDisallowed" +if ((-not $inRoot) -or $inDisallowed) { + Record-Check 'Root anchor not trusted' 'PASS' $storeSummary +} else { + Record-Check 'Root anchor not trusted' 'FAIL' "Present in LocalMachine\Root and not in Disallowed ($storeSummary)" + $GatingFailures++ +} + +# 2. OS-store client (Invoke-WebRequest) rejects the test endpoint (only a successful connection fails the job) +$progTmp = $ProgressPreference +$ProgressPreference = 'SilentlyContinue' try { - $rootCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Root' -ErrorAction Stop) - $inRoot = $false - foreach ($cert in $rootCerts) { - $hashBytes = $sha256.ComputeHash($cert.RawData) - $hex = [System.BitConverter]::ToString($hashBytes).Replace('-', '').ToUpperInvariant() - if ($hex -eq $EmSignG3Hex) { - $inRoot = $true + $null = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -TimeoutSec 15 + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded (certificate was trusted)" + $GatingFailures++ +} catch { + $certRejected = $false + $hostnameMismatchOnly = $false + $ex = $_.Exception + while ($null -ne $ex) { + if ( + ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or + ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure) + ) { + $certRejected = $true break } - } - - $inDisallowed = $false - if (Test-Path -Path 'Cert:\LocalMachine\Disallowed') { - $disallowedCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Disallowed' -ErrorAction SilentlyContinue) - foreach ($cert in $disallowedCerts) { - $hashBytes = $sha256.ComputeHash($cert.RawData) - $hex = [System.BitConverter]::ToString($hashBytes).Replace('-', '').ToUpperInvariant() - if ($hex -eq $EmSignG3Hex) { - $inDisallowed = $true - break - } + if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') { + $hostnameMismatchOnly = $true } + $ex = $ex.InnerException } - - $storeSummary = "LocalMachine\Root=$inRoot ($($rootCerts.Count) certs), LocalMachine\Disallowed=$inDisallowed" - if ((-not $inRoot) -or $inDisallowed) { - Record-Check 'Root anchor not trusted' 'PASS' $storeSummary - } else { - Record-Check 'Root anchor not trusted' 'FAIL' "Present in LocalMachine\Root and not in Disallowed ($storeSummary)" + $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|') + if ($certRejected) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with certificate validation error ($cleanMsg)" + } elseif ($hostnameMismatchOnly) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)" $GatingFailures++ + } else { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)" } } finally { - $sha256.Dispose() -} - -# 2. OS-store client (Invoke-WebRequest) rejects the test endpoint (only a successful connection fails the job) -if ([string]::IsNullOrEmpty($TestEndpoint)) { - Record-Check 'Invoke-WebRequest rejects test endpoint' 'SKIP' 'Skipped via empty `TRUST_PROBE_TEST_ENDPOINT`' -} else { - $progTmp = $ProgressPreference - $ProgressPreference = 'SilentlyContinue' - try { - $null = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -TimeoutSec 15 - Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded (certificate was trusted)" - $GatingFailures++ - } catch { - $certRejected = $false - $hostnameMismatchOnly = $false - $ex = $_.Exception - while ($null -ne $ex) { - if ( - ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or - ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure) - ) { - $certRejected = $true - break - } - if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') { - $hostnameMismatchOnly = $true - } - $ex = $ex.InnerException - } - $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|') - if ($certRejected) { - Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with certificate validation error ($cleanMsg)" - } elseif ($hostnameMismatchOnly) { - Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)" - $GatingFailures++ - } else { - Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)" - } - } finally { - $ProgressPreference = $progTmp - } + $ProgressPreference = $progTmp } Append-Summary "" diff --git a/ci/tools/runner_trust_probe.sh b/ci/tools/runner_trust_probe.sh index 126f1c4cc59..f87a51bedcd 100755 --- a/ci/tools/runner_trust_probe.sh +++ b/ci/tools/runner_trust_probe.sh @@ -6,16 +6,14 @@ # runner_trust_probe.sh -- probe the OS certificate store on a Linux runner # for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3). # -# Optional environment overrides for local testing: -# TRUST_PROBE_CA_BUNDLE Path to CA bundle (default: /etc/ssl/certs/ca-certificates.crt) -# TRUST_PROBE_TEST_ENDPOINT HTTPS endpoint to test (default: https://dv-g3b-emudhra.we-have-a-problem.com/) -# TRUST_PROBE_LABEL Heading label for GITHUB_STEP_SUMMARY +# Optional environment variable: +# TRUST_PROBE_LABEL Heading label for GITHUB_STEP_SUMMARY set -euo pipefail EMSIGN_G3_SHA256="86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B" -CA_BUNDLE="${TRUST_PROBE_CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" -TEST_ENDPOINT="${TRUST_PROBE_TEST_ENDPOINT:-https://dv-g3b-emudhra.we-have-a-problem.com/}" +CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" +TEST_ENDPOINT="https://dv-g3b-emudhra.we-have-a-problem.com/" PROBE_LABEL="${TRUST_PROBE_LABEL:-Linux OS trust store}" summary_file="${GITHUB_STEP_SUMMARY:-}" @@ -48,75 +46,34 @@ elif ! command -v openssl >/dev/null 2>&1; then record_check "Root anchor absent" "FAIL" "\`openssl\` not found on PATH" gating_failures=$((gating_failures + 1)) else - tmp_dir="$(mktemp -d)" - trap 'rm -rf "${tmp_dir}"' EXIT - - awk -v dir="${tmp_dir}" ' - /-----BEGIN CERTIFICATE-----/ { in_cert = 1; idx++; file = sprintf("%s/cert-%04d.pem", dir, idx) } - in_cert { print > file } - /-----END CERTIFICATE-----/ { in_cert = 0; close(file) } - ' "${CA_BUNDLE}" - - shopt -s nullglob - cert_files=("${tmp_dir}"/cert-*.pem) - shopt -u nullglob - - if (( ${#cert_files[@]} == 0 )); then + fingerprints="$(awk ' + /-----BEGIN CERTIFICATE-----/ { cert = "" } + { cert = cert $0 "\n" } + /-----END CERTIFICATE-----/ { + cmd = "openssl x509 -noout -fingerprint -sha256 2>/dev/null" + printf "%s", cert | cmd + close(cmd) + } + ' "${CA_BUNDLE}")" + cert_count="$(grep -c . <<< "${fingerprints}" || true)" + + if (( cert_count == 0 )); then record_check "Root anchor absent" "FAIL" "No certificates parsed from \`${CA_BUNDLE}\`" gating_failures=$((gating_failures + 1)) + elif grep -qiF "${EMSIGN_G3_SHA256}" <<< "${fingerprints}"; then + record_check "Root anchor absent" "FAIL" "Found \`emSign ECC Root CA - G3\` (\`${EMSIGN_G3_SHA256}\`) in \`${CA_BUNDLE}\` (${cert_count} certs checked)" + gating_failures=$((gating_failures + 1)) else - found_emsign=0 - for cert_file in "${cert_files[@]}"; do - fp="$(openssl x509 -in "${cert_file}" -noout -fingerprint -sha256 2>/dev/null | sed 's/^.*=//; s/[[:space:]]//g' | tr '[:lower:]' '[:upper:]')" - if [[ "${fp}" == "${EMSIGN_G3_SHA256}" ]]; then - found_emsign=1 - break - fi - done - - if (( found_emsign == 1 )); then - record_check "Root anchor absent" "FAIL" "Found \`emSign ECC Root CA - G3\` (\`${EMSIGN_G3_SHA256}\`) in \`${CA_BUNDLE}\` (${#cert_files[@]} certs checked)" - gating_failures=$((gating_failures + 1)) - else - record_check "Root anchor absent" "PASS" "Absent from \`${CA_BUNDLE}\` (${#cert_files[@]} certs checked)" - fi + record_check "Root anchor absent" "PASS" "Absent from \`${CA_BUNDLE}\` (${cert_count} certs checked)" fi fi # 2. ca-certificates package version (informational) ca_pkg_ver="$(dpkg-query -W -f='${Version}' ca-certificates 2>/dev/null || echo 'not-installed')" -distro_id="unknown" -distro_ver="unknown" -if [[ -f /etc/os-release ]]; then - # shellcheck disable=SC1091 - source /etc/os-release - distro_id="${ID:-unknown}" - distro_ver="${VERSION_ID:-unknown}" -fi - -case "${distro_id}:${distro_ver}" in - ubuntu:22.04) expected_min="20260816ubuntu0.22.04.1" ;; - ubuntu:24.04) expected_min="20260816ubuntu0.24.04.1" ;; - ubuntu:26.04) expected_min="20260816ubuntu0.26.04.1" ;; - *) expected_min="" ;; -esac - -if [[ "${ca_pkg_ver}" == "not-installed" ]]; then - record_check "ca-certificates version" "INFO" "Package not installed (distro: \`${distro_id} ${distro_ver}\`)" -elif [[ -n "${expected_min}" ]]; then - if dpkg --compare-versions "${ca_pkg_ver}" ge "${expected_min}"; then - record_check "ca-certificates version" "INFO (OK)" "\`${ca_pkg_ver}\` >= \`${expected_min}\` (Ubuntu ${distro_ver})" - else - record_check "ca-certificates version" "INFO (STALE)" "\`${ca_pkg_ver}\` < \`${expected_min}\` (Ubuntu ${distro_ver})" - fi -else - record_check "ca-certificates version" "INFO" "\`${ca_pkg_ver}\` on \`${distro_id} ${distro_ver}\`" -fi +record_check "ca-certificates version" "INFO" "\`${ca_pkg_ver}\`" # 3. OS-store client (curl) rejects the test endpoint (only a successful TLS connection fails the job) -if [[ -z "${TEST_ENDPOINT}" ]]; then - record_check "curl rejects test endpoint" "SKIP" "Skipped via empty \`TRUST_PROBE_TEST_ENDPOINT\`" -elif ! command -v curl >/dev/null 2>&1; then +if ! command -v curl >/dev/null 2>&1; then record_check "curl rejects test endpoint" "INCONCLUSIVE" "\`curl\` not found on PATH" else curl_rc=0 diff --git a/ci/tools/runner_trust_probe_node.js b/ci/tools/runner_trust_probe_node.js index 5afa460f3d4..d6df97810a3 100644 --- a/ci/tools/runner_trust_probe_node.js +++ b/ci/tools/runner_trust_probe_node.js @@ -9,7 +9,11 @@ const https = require('https'); -module.exports = async function runNodeTrustProbe({ core }) { +module.exports = async function runNodeTrustProbe({ + core, + heading = 'Node runtime trust store', + gating = true, +}) { const version = process.version; const execPath = process.execPath; const nodeOptions = process.env.NODE_OPTIONS || ''; @@ -20,7 +24,7 @@ module.exports = async function runNodeTrustProbe({ core }) { const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); - const endpoint = process.env.TRUST_PROBE_TEST_ENDPOINT || 'https://dv-g3b-emudhra.we-have-a-problem.com/'; + const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; const probeEndpoint = () => new Promise((resolve) => { const req = https.get(endpoint, { timeout: 15000 }, (res) => { res.resume(); @@ -50,14 +54,14 @@ module.exports = async function runNodeTrustProbe({ core }) { ]; await core.summary - .addHeading('Node runtime trust store', 3) + .addHeading(heading, 3) .addTable([ [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], ...rows, ]) .write(); - if (!versionOk || tlsResult.status === 'FAIL') { + if (gating && (!versionOk || tlsResult.status === 'FAIL')) { core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); } }; From ca21deb23803c2c3e0a1c45baea7b262c50e5255 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Tue, 27 Oct 2026 15:03:21 -0700 Subject: [PATCH 10/10] CI: use blobless checkout in trust probe and pass -SkipHttpErrorCheck on Windows Signed-off-by: Ralf Juengling --- .github/workflows/runner-trust-probe.yml | 9 +++++++-- ci/tools/runner_trust_probe.ps1 | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml index 7a974d27ad9..c1fb0dd9f36 100644 --- a/.github/workflows/runner-trust-probe.yml +++ b/.github/workflows/runner-trust-probe.yml @@ -2,8 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 # -# Report-only probe of OS and bundled Node.js certificate stores across hosted -# and self-hosted runner labels for the eMudhra/emSign CA incident. +# Temporary, report-only probe of OS and bundled Node.js certificate stores +# across hosted and self-hosted runner labels for the eMudhra/emSign CA +# incident. Delete this workflow and ci/tools/runner_trust_probe* once all +# runner labels pass and the rollout is complete. name: "CI: Runner trust probe" @@ -39,6 +41,7 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + filter: blob:none persist-credentials: false - name: Compute runner labels @@ -89,6 +92,7 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + filter: blob:none persist-credentials: false - name: Probe Linux OS trust store @@ -147,6 +151,7 @@ jobs: - name: Checkout ${{ github.event.repository.name }} uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + filter: blob:none persist-credentials: false - name: Setup proxy cache diff --git a/ci/tools/runner_trust_probe.ps1 b/ci/tools/runner_trust_probe.ps1 index 8bf103d6d6f..4668b1ea4d5 100644 --- a/ci/tools/runner_trust_probe.ps1 +++ b/ci/tools/runner_trust_probe.ps1 @@ -48,8 +48,8 @@ if ((-not $inRoot) -or $inDisallowed) { $progTmp = $ProgressPreference $ProgressPreference = 'SilentlyContinue' try { - $null = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -TimeoutSec 15 - Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded (certificate was trusted)" + $res = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -SkipHttpErrorCheck -TimeoutSec 15 + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded with HTTP $($res.StatusCode) (certificate was trusted)" $GatingFailures++ } catch { $certRejected = $false