[no-ci] CI: add runner certificate trust store probe workflow

.github/workflows/runner-trust-probe.yml

@@ -0,0 +1,187 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Temporary, report-only probe of OS and bundled Node.js certificate stores
+# across hosted and self-hosted runner labels for the eMudhra/emSign CA
+# incident. Delete this workflow and ci/tools/runner_trust_probe* once all
+# runner labels pass and the rollout is complete.
+
+name: "CI: Runner trust probe"
+
+on:
+  schedule:
+    # Weekly on Mondays at 06:23 UTC
+    - cron: "23 6 * * 1"
+  workflow_dispatch:
+    inputs:
+      label-filter:
+        description: "Optional regex to filter runner labels (e.g. '^(ubuntu-latest|linux-amd64-cpu8)$'; empty probes all)"
+        required: false
+        default: ""
+        type: string
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash --noprofile --norc -xeuo pipefail {0}
+
+jobs:
+  compute-matrix:
+    name: Compute runner label sets
+    if: ${{ github.repository_owner == 'nvidia' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      host: ${{ steps.compute.outputs.host }}
+      linux_gpu: ${{ steps.compute.outputs.linux_gpu }}
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          filter: blob:none
+          persist-credentials: false
+
+      - name: Compute runner labels
+        id: compute
+        env:
+          LABEL_FILTER: ${{ inputs.label-filter || '' }}
+        run: |
+          # Hosted and self-hosted CPU labels are listed explicitly; GPU labels
+          # are derived from ci/test-matrix.yml.
+          labels_json="$(yq -o json '.' ci/test-matrix.yml | jq -c --arg filter "${LABEL_FILTER}" '
+            {
+              host: (
+                [
+                  "ubuntu-latest",
+                  "linux-amd64-cpu8",
+                  "linux-arm64-cpu8",
+                  "linux-amd64-cpu4",
+                  "windows-latest",
+                  "windows-2022",
+                  "windows-11-arm"
+                ] + [
+                  (.windows["pull-request"][]?, .windows["nightly"][]?)
+                  | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)"
+                ]
+              ),
+              linux_gpu: [
+                (.linux["pull-request"][]?, .linux["nightly"][]?)
+                | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)"
+              ]
+            }
+            | map_values(unique | if $filter != "" then map(select(test($filter))) else . end)
+          ')"
+
+          echo "host=$(jq -c '.host' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}"
+          echo "linux_gpu=$(jq -c '.linux_gpu' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}"
+
+  probe-host:
+    name: Probe (${{ matrix.label }})
+    needs: compute-matrix
+    if: ${{ needs.compute-matrix.outputs.host != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        label: ${{ fromJSON(needs.compute-matrix.outputs.host) }}
+    runs-on: ${{ matrix.label }}
+    timeout-minutes: 15
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          filter: blob:none
+          persist-credentials: false
+
+      - name: Probe Linux OS trust store
+        if: ${{ runner.os != 'Windows' }}
+        env:
+          TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}"
+        run: ./ci/tools/runner_trust_probe.sh
+
+      - name: Probe Linux OS trust store after ca-certificates upgrade (ubuntu-latest)
+        if: ${{ always() && matrix.label == 'ubuntu-latest' }}
+        env:
+          TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)"
+        run: |
+          sudo apt-get update
+          sudo apt-get install --only-upgrade -y ca-certificates
+          ./ci/tools/runner_trust_probe.sh
+
+      - name: Probe Windows OS trust store
+        if: ${{ runner.os == 'Windows' }}
+        shell: pwsh
+        run: ./ci/tools/runner_trust_probe.ps1
+
+      - name: Probe Node 20 action runtime (informational)
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1 (declares node20)
+        with:
+          script: |
+            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
+            await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false });
+
+      - name: Probe bundled Node runtime and trust store
+        if: ${{ always() }}
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
+        with:
+          script: |
+            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
+            await probe({ core });
+
+  probe-linux-gpu:
+    name: Probe (${{ matrix.label }})
+    needs: compute-matrix
+    if: ${{ needs.compute-matrix.outputs.linux_gpu != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        label: ${{ fromJSON(needs.compute-matrix.outputs.linux_gpu) }}
+    runs-on: ${{ matrix.label }}
+    timeout-minutes: 15
+    # Match test-wheel-linux.yml container setup so the probe inspects the exact
+    # container environment that Linux GPU test jobs see.
+    container:
+      options: -u root --security-opt seccomp=unconfined --shm-size 16g
+      image: ubuntu:24.04
+    steps:
+      - name: Checkout ${{ github.event.repository.name }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          filter: blob:none
+          persist-credentials: false
+
+      - name: Setup proxy cache
+        uses: nv-gha-runners/setup-proxy-cache@main
+        continue-on-error: true
+
+      - name: Install container CA certificates and probe tools
+        uses: ./.github/actions/install_unix_deps
+        with:
+          dependencies: "ca-certificates curl openssl"
+          dependent_exes: "curl openssl"
+
+      - name: Probe container OS trust store
+        env:
+          TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)"
+        run: ./ci/tools/runner_trust_probe.sh
+
+      - name: Probe Node 20 action runtime (informational)
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea  # v7.0.1 (declares node20)
+        with:
+          script: |
+            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
+            await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false });
+
+      - name: Probe bundled Node runtime and trust store
+        if: ${{ always() }}
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
+        with:
+          script: |
+            const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
+            await probe({ core });

ci/tools/compute_ci_plan.py

@@ -37,10 +37,14 @@
 IGNORED_SUFFIXES = {".md", ".svg"}
 IGNORED_PATHS = {
     ".coveragerc",
+    ".github/workflows/runner-trust-probe.yml",
     ".gitignore",
     ".pre-commit-config.yaml",
     ".spdx-ignore",
     "LICENSE",
+    "ci/tools/runner_trust_probe.ps1",
+    "ci/tools/runner_trust_probe.sh",
+    "ci/tools/runner_trust_probe_node.js",
     "context7.json",
     "greptile.json",
     "ruff.toml",
@@ -86,6 +90,8 @@ def compute_workplan(
             if path_parts[0] == "ci" or (
                 len(path_parts) >= 2 and path_parts[:2] in {(".github", "actions"), (".github", "workflows")}
             ):
+                if path in IGNORED_PATHS:
+                    continue
                 force_all = True
                 break
 

ci/tools/runner_trust_probe.ps1

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# runner_trust_probe.ps1 -- probe the Windows certificate store on a CI runner
+# for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3).
+
+$ErrorActionPreference = 'Stop'
+
+$EmSignG3Hex = '86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B'
+$TestEndpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'
+$ProbeLabel = 'Windows OS trust store'
+$SummaryFile = $env:GITHUB_STEP_SUMMARY
+
+function Append-Summary([string]$Line) {
+    if ($SummaryFile) {
+        Add-Content -Path $SummaryFile -Value $Line -Encoding utf8
+    }
+}
+
+function Record-Check([string]$Name, [string]$Status, [string]$Details) {
+    Write-Output "[$Status] ${Name}: $Details"
+    Append-Summary "| $Name | **$Status** | $Details |"
+}
+
+$GatingFailures = 0
+
+Append-Summary "### $ProbeLabel"
+Append-Summary ""
+Append-Summary "| Check | Status | Details |"
+Append-Summary "| --- | --- | --- |"
+
+# 1. Root anchor not trusted in Cert:\LocalMachine\Root (or present in Disallowed) (gating)
+$rootCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Root' -ErrorAction Stop)
+$inRoot = [bool]($rootCerts | Where-Object { $_.GetCertHashString('SHA256') -eq $EmSignG3Hex })
+$inDisallowed = [bool](Get-ChildItem -Path 'Cert:\LocalMachine\Disallowed' -ErrorAction SilentlyContinue |
+    Where-Object { $_.GetCertHashString('SHA256') -eq $EmSignG3Hex })
+
+$storeSummary = "LocalMachine\Root=$inRoot ($($rootCerts.Count) certs), LocalMachine\Disallowed=$inDisallowed"
+if ((-not $inRoot) -or $inDisallowed) {
+    Record-Check 'Root anchor not trusted' 'PASS' $storeSummary
+} else {
+    Record-Check 'Root anchor not trusted' 'FAIL' "Present in LocalMachine\Root and not in Disallowed ($storeSummary)"
+    $GatingFailures++
+}
+
+# 2. OS-store client (Invoke-WebRequest) rejects the test endpoint (only a successful connection fails the job)
+$progTmp = $ProgressPreference
+$ProgressPreference = 'SilentlyContinue'
+try {
+    $res = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -SkipHttpErrorCheck -TimeoutSec 15
+    Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded with HTTP $($res.StatusCode) (certificate was trusted)"
+    $GatingFailures++
+} catch {
+    $certRejected = $false
+    $hostnameMismatchOnly = $false
+    $ex = $_.Exception
+    while ($null -ne $ex) {
+        if (
+            ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or
+            ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure)
+        ) {
+            $certRejected = $true
+            break
+        }
+        if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') {
+            $hostnameMismatchOnly = $true
+        }
+        $ex = $ex.InnerException
+    }
+    $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|')
+    if ($certRejected) {
+        Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with certificate validation error ($cleanMsg)"
+    } elseif ($hostnameMismatchOnly) {
+        Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)"
+        $GatingFailures++
+    } else {
+        Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)"
+    }
+} finally {
+    $ProgressPreference = $progTmp
+}
+
+Append-Summary ""
+
+if ($GatingFailures -gt 0) {
+    Write-Error "${ProbeLabel}: $GatingFailures gating check(s) failed"
+    exit 1
+}