diff --git a/.github/workflows/runner-trust-probe.yml b/.github/workflows/runner-trust-probe.yml new file mode 100644 index 00000000000..c1fb0dd9f36 --- /dev/null +++ b/.github/workflows/runner-trust-probe.yml @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Temporary, report-only probe of OS and bundled Node.js certificate stores +# across hosted and self-hosted runner labels for the eMudhra/emSign CA +# incident. Delete this workflow and ci/tools/runner_trust_probe* once all +# runner labels pass and the rollout is complete. + +name: "CI: Runner trust probe" + +on: + schedule: + # Weekly on Mondays at 06:23 UTC + - cron: "23 6 * * 1" + workflow_dispatch: + inputs: + label-filter: + description: "Optional regex to filter runner labels (e.g. '^(ubuntu-latest|linux-amd64-cpu8)$'; empty probes all)" + required: false + default: "" + type: string + +permissions: + contents: read + +defaults: + run: + shell: bash --noprofile --norc -xeuo pipefail {0} + +jobs: + compute-matrix: + name: Compute runner label sets + if: ${{ github.repository_owner == 'nvidia' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + host: ${{ steps.compute.outputs.host }} + linux_gpu: ${{ steps.compute.outputs.linux_gpu }} + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + filter: blob:none + persist-credentials: false + + - name: Compute runner labels + id: compute + env: + LABEL_FILTER: ${{ inputs.label-filter || '' }} + run: | + # Hosted and self-hosted CPU labels are listed explicitly; GPU labels + # are derived from ci/test-matrix.yml. + labels_json="$(yq -o json '.' ci/test-matrix.yml | jq -c --arg filter "${LABEL_FILTER}" ' + { + host: ( + [ + "ubuntu-latest", + "linux-amd64-cpu8", + "linux-arm64-cpu8", + "linux-amd64-cpu4", + "windows-latest", + "windows-2022", + "windows-11-arm" + ] + [ + (.windows["pull-request"][]?, .windows["nightly"][]?) + | "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" + ] + ), + linux_gpu: [ + (.linux["pull-request"][]?, .linux["nightly"][]?) + | "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)" + ] + } + | map_values(unique | if $filter != "" then map(select(test($filter))) else . end) + ')" + + echo "host=$(jq -c '.host' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}" + echo "linux_gpu=$(jq -c '.linux_gpu' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}" + + probe-host: + name: Probe (${{ matrix.label }}) + needs: compute-matrix + if: ${{ needs.compute-matrix.outputs.host != '[]' }} + strategy: + fail-fast: false + matrix: + label: ${{ fromJSON(needs.compute-matrix.outputs.host) }} + runs-on: ${{ matrix.label }} + timeout-minutes: 15 + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + filter: blob:none + persist-credentials: false + + - name: Probe Linux OS trust store + if: ${{ runner.os != 'Windows' }} + env: + TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}" + run: ./ci/tools/runner_trust_probe.sh + + - name: Probe Linux OS trust store after ca-certificates upgrade (ubuntu-latest) + if: ${{ always() && matrix.label == 'ubuntu-latest' }} + env: + TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)" + run: | + sudo apt-get update + sudo apt-get install --only-upgrade -y ca-certificates + ./ci/tools/runner_trust_probe.sh + + - name: Probe Windows OS trust store + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: ./ci/tools/runner_trust_probe.ps1 + + - name: Probe Node 20 action runtime (informational) + if: ${{ always() }} + continue-on-error: true + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) + with: + script: | + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false }); + + - name: Probe bundled Node runtime and trust store + if: ${{ always() }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core }); + + probe-linux-gpu: + name: Probe (${{ matrix.label }}) + needs: compute-matrix + if: ${{ needs.compute-matrix.outputs.linux_gpu != '[]' }} + strategy: + fail-fast: false + matrix: + label: ${{ fromJSON(needs.compute-matrix.outputs.linux_gpu) }} + runs-on: ${{ matrix.label }} + timeout-minutes: 15 + # Match test-wheel-linux.yml container setup so the probe inspects the exact + # container environment that Linux GPU test jobs see. + container: + options: -u root --security-opt seccomp=unconfined --shm-size 16g + image: ubuntu:24.04 + steps: + - name: Checkout ${{ github.event.repository.name }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + filter: blob:none + persist-credentials: false + + - name: Setup proxy cache + uses: nv-gha-runners/setup-proxy-cache@main + continue-on-error: true + + - name: Install container CA certificates and probe tools + uses: ./.github/actions/install_unix_deps + with: + dependencies: "ca-certificates curl openssl" + dependent_exes: "curl openssl" + + - name: Probe container OS trust store + env: + TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)" + run: ./ci/tools/runner_trust_probe.sh + + - name: Probe Node 20 action runtime (informational) + if: ${{ always() }} + continue-on-error: true + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20) + with: + script: | + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false }); + + - name: Probe bundled Node runtime and trust store + if: ${{ always() }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`); + await probe({ core }); diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py index ea03abc31a9..51493044569 100644 --- a/ci/tools/compute_ci_plan.py +++ b/ci/tools/compute_ci_plan.py @@ -37,10 +37,14 @@ IGNORED_SUFFIXES = {".md", ".svg"} IGNORED_PATHS = { ".coveragerc", + ".github/workflows/runner-trust-probe.yml", ".gitignore", ".pre-commit-config.yaml", ".spdx-ignore", "LICENSE", + "ci/tools/runner_trust_probe.ps1", + "ci/tools/runner_trust_probe.sh", + "ci/tools/runner_trust_probe_node.js", "context7.json", "greptile.json", "ruff.toml", @@ -86,6 +90,8 @@ def compute_workplan( if path_parts[0] == "ci" or ( len(path_parts) >= 2 and path_parts[:2] in {(".github", "actions"), (".github", "workflows")} ): + if path in IGNORED_PATHS: + continue force_all = True break diff --git a/ci/tools/runner_trust_probe.ps1 b/ci/tools/runner_trust_probe.ps1 new file mode 100644 index 00000000000..4668b1ea4d5 --- /dev/null +++ b/ci/tools/runner_trust_probe.ps1 @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# runner_trust_probe.ps1 -- probe the Windows certificate store on a CI runner +# for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3). + +$ErrorActionPreference = 'Stop' + +$EmSignG3Hex = '86A1ECBA089C4A8D3BBE2734C612BA341D813E043CF9E8A862CD5C57A36BBE6B' +$TestEndpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/' +$ProbeLabel = 'Windows OS trust store' +$SummaryFile = $env:GITHUB_STEP_SUMMARY + +function Append-Summary([string]$Line) { + if ($SummaryFile) { + Add-Content -Path $SummaryFile -Value $Line -Encoding utf8 + } +} + +function Record-Check([string]$Name, [string]$Status, [string]$Details) { + Write-Output "[$Status] ${Name}: $Details" + Append-Summary "| $Name | **$Status** | $Details |" +} + +$GatingFailures = 0 + +Append-Summary "### $ProbeLabel" +Append-Summary "" +Append-Summary "| Check | Status | Details |" +Append-Summary "| --- | --- | --- |" + +# 1. Root anchor not trusted in Cert:\LocalMachine\Root (or present in Disallowed) (gating) +$rootCerts = @(Get-ChildItem -Path 'Cert:\LocalMachine\Root' -ErrorAction Stop) +$inRoot = [bool]($rootCerts | Where-Object { $_.GetCertHashString('SHA256') -eq $EmSignG3Hex }) +$inDisallowed = [bool](Get-ChildItem -Path 'Cert:\LocalMachine\Disallowed' -ErrorAction SilentlyContinue | + Where-Object { $_.GetCertHashString('SHA256') -eq $EmSignG3Hex }) + +$storeSummary = "LocalMachine\Root=$inRoot ($($rootCerts.Count) certs), LocalMachine\Disallowed=$inDisallowed" +if ((-not $inRoot) -or $inDisallowed) { + Record-Check 'Root anchor not trusted' 'PASS' $storeSummary +} else { + Record-Check 'Root anchor not trusted' 'FAIL' "Present in LocalMachine\Root and not in Disallowed ($storeSummary)" + $GatingFailures++ +} + +# 2. OS-store client (Invoke-WebRequest) rejects the test endpoint (only a successful connection fails the job) +$progTmp = $ProgressPreference +$ProgressPreference = 'SilentlyContinue' +try { + $res = Invoke-WebRequest -Uri $TestEndpoint -UseBasicParsing -SkipHttpErrorCheck -TimeoutSec 15 + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Connection to ``$TestEndpoint`` succeeded with HTTP $($res.StatusCode) (certificate was trusted)" + $GatingFailures++ +} catch { + $certRejected = $false + $hostnameMismatchOnly = $false + $ex = $_.Exception + while ($null -ne $ex) { + if ( + ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'certificate chain|UntrustedRoot|RemoteCertificateChainErrors') -or + ($ex -is [System.Net.WebException] -and $ex.Status -eq [System.Net.WebExceptionStatus]::TrustFailure) + ) { + $certRejected = $true + break + } + if ($ex -is [System.Security.Authentication.AuthenticationException] -and $ex.Message -match 'RemoteCertificateNameMismatch') { + $hostnameMismatchOnly = $true + } + $ex = $ex.InnerException + } + $cleanMsg = ($_.Exception.Message -replace '[\r\n]+', ' ' -replace '\|', '\|') + if ($certRejected) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'PASS' "Rejected with certificate validation error ($cleanMsg)" + } elseif ($hostnameMismatchOnly) { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'FAIL' "Certificate chain was trusted; failed only hostname check ($cleanMsg)" + $GatingFailures++ + } else { + Record-Check 'Invoke-WebRequest rejects test endpoint' 'INCONCLUSIVE' "Non-certificate or network error ($cleanMsg)" + } +} finally { + $ProgressPreference = $progTmp +} + +Append-Summary "" + +if ($GatingFailures -gt 0) { + Write-Error "${ProbeLabel}: $GatingFailures gating check(s) failed" + exit 1 +} diff --git a/ci/tools/runner_trust_probe.sh b/ci/tools/runner_trust_probe.sh new file mode 100755 index 00000000000..f87a51bedcd --- /dev/null +++ b/ci/tools/runner_trust_probe.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# runner_trust_probe.sh -- probe the OS certificate store on a Linux runner +# for the compromised eMudhra/emSign root anchor (emSign ECC Root CA - G3). +# +# Optional environment variable: +# TRUST_PROBE_LABEL Heading label for GITHUB_STEP_SUMMARY + +set -euo pipefail + +EMSIGN_G3_SHA256="86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B" +CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" +TEST_ENDPOINT="https://dv-g3b-emudhra.we-have-a-problem.com/" +PROBE_LABEL="${TRUST_PROBE_LABEL:-Linux OS trust store}" + +summary_file="${GITHUB_STEP_SUMMARY:-}" +append_summary() { + if [[ -n "${summary_file}" ]]; then + printf '%s\n' "$*" >> "${summary_file}" + fi +} + +gating_failures=0 + +append_summary "### ${PROBE_LABEL}" +append_summary "" +append_summary "| Check | Status | Details |" +append_summary "| --- | --- | --- |" + +record_check() { + local name="$1" + local status="$2" + local details="$3" + printf '[%s] %s: %s\n' "${status}" "${name}" "${details}" + append_summary "| ${name} | **${status}** | ${details} |" +} + +# 1. Root anchor absent from CA bundle (gating) +if [[ ! -f "${CA_BUNDLE}" ]]; then + record_check "Root anchor absent" "FAIL" "Bundle not found: \`${CA_BUNDLE}\`" + gating_failures=$((gating_failures + 1)) +elif ! command -v openssl >/dev/null 2>&1; then + record_check "Root anchor absent" "FAIL" "\`openssl\` not found on PATH" + gating_failures=$((gating_failures + 1)) +else + fingerprints="$(awk ' + /-----BEGIN CERTIFICATE-----/ { cert = "" } + { cert = cert $0 "\n" } + /-----END CERTIFICATE-----/ { + cmd = "openssl x509 -noout -fingerprint -sha256 2>/dev/null" + printf "%s", cert | cmd + close(cmd) + } + ' "${CA_BUNDLE}")" + cert_count="$(grep -c . <<< "${fingerprints}" || true)" + + if (( cert_count == 0 )); then + record_check "Root anchor absent" "FAIL" "No certificates parsed from \`${CA_BUNDLE}\`" + gating_failures=$((gating_failures + 1)) + elif grep -qiF "${EMSIGN_G3_SHA256}" <<< "${fingerprints}"; then + record_check "Root anchor absent" "FAIL" "Found \`emSign ECC Root CA - G3\` (\`${EMSIGN_G3_SHA256}\`) in \`${CA_BUNDLE}\` (${cert_count} certs checked)" + gating_failures=$((gating_failures + 1)) + else + record_check "Root anchor absent" "PASS" "Absent from \`${CA_BUNDLE}\` (${cert_count} certs checked)" + fi +fi + +# 2. ca-certificates package version (informational) +ca_pkg_ver="$(dpkg-query -W -f='${Version}' ca-certificates 2>/dev/null || echo 'not-installed')" +record_check "ca-certificates version" "INFO" "\`${ca_pkg_ver}\`" + +# 3. OS-store client (curl) rejects the test endpoint (only a successful TLS connection fails the job) +if ! command -v curl >/dev/null 2>&1; then + record_check "curl rejects test endpoint" "INCONCLUSIVE" "\`curl\` not found on PATH" +else + curl_rc=0 + curl_err="$(curl -sS --max-time 15 --cacert "${CA_BUNDLE}" -o /dev/null "${TEST_ENDPOINT}" 2>&1)" || curl_rc=$? + curl_err_oneline="$(tr '\n' ' ' <<< "${curl_err}" | sed 's/|/\\|/g')" + if (( curl_rc == 0 )); then + record_check "curl rejects test endpoint" "FAIL" "Connection to \`${TEST_ENDPOINT}\` succeeded (certificate was trusted)" + gating_failures=$((gating_failures + 1)) + elif (( curl_rc == 60 )); then + if grep -qiE 'no alternative certificate subject name|subject alternative name|does not match' <<< "${curl_err}"; then + record_check "curl rejects test endpoint" "FAIL" "Certificate chain was trusted; failed only hostname check (${curl_err_oneline})" + gating_failures=$((gating_failures + 1)) + else + record_check "curl rejects test endpoint" "PASS" "Rejected with curl exit 60 (peer certificate cannot be authenticated)" + fi + else + record_check "curl rejects test endpoint" "INCONCLUSIVE" "curl exited ${curl_rc} (network/DNS or non-verification error: ${curl_err_oneline})" + fi +fi + +append_summary "" + +if (( gating_failures > 0 )); then + echo "::error::${PROBE_LABEL}: ${gating_failures} gating check(s) failed" >&2 + exit 1 +fi diff --git a/ci/tools/runner_trust_probe_node.js b/ci/tools/runner_trust_probe_node.js new file mode 100644 index 00000000000..d6df97810a3 --- /dev/null +++ b/ci/tools/runner_trust_probe_node.js @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// +// runner_trust_probe_node.js -- probe the runner agent's bundled Node.js +// version and certificate store for the eMudhra/emSign CA incident. + +'use strict'; + +const https = require('https'); + +module.exports = async function runNodeTrustProbe({ + core, + heading = 'Node runtime trust store', + gating = true, +}) { + const version = process.version; + const execPath = process.execPath; + const nodeOptions = process.env.NODE_OPTIONS || ''; + const extraCa = process.env.NODE_EXTRA_CA_CERTS || ''; + const useSystemCa = process.env.NODE_USE_SYSTEM_CA || ''; + + const m = /^v(\d+)\.(\d+)\.(\d+)/.exec(version); + const [major, minor, patch] = m ? [Number(m[1]), Number(m[2]), Number(m[3])] : [0, 0, 0]; + const versionOk = major === 24 && (minor > 22 || (minor === 22 && patch >= 1)); + + const endpoint = 'https://dv-g3b-emudhra.we-have-a-problem.com/'; + const probeEndpoint = () => new Promise((resolve) => { + const req = https.get(endpoint, { timeout: 15000 }, (res) => { + res.resume(); + resolve({ status: 'FAIL', detail: `Connected with HTTP ${res.statusCode} (certificate was trusted)` }); + }); + req.on('timeout', () => { + req.destroy(new Error('Request timed out')); + }); + req.on('error', (err) => { + const code = err.code || ''; + const msg = `${code} ${err.message || ''}`.trim(); + if (code === 'ERR_TLS_CERT_ALTNAME_INVALID') { + resolve({ status: 'FAIL', detail: `Certificate chain was trusted; failed only hostname check (${msg})` }); + } else if (/^(CERT_|UNABLE_TO_|SELF_SIGNED_|DEPTH_ZERO_)/.test(code)) { + resolve({ status: 'PASS', detail: `Rejected with certificate error (${msg})` }); + } else { + resolve({ status: 'INCONCLUSIVE', detail: `Network/DNS or non-certificate error (${msg})` }); + } + }); + }); + + const tlsResult = await probeEndpoint(); + const rows = [ + ['Node version', versionOk ? '**PASS**' : '**FAIL**', `\`${version}\` (require Node 24 \`>= v24.22.1\`; \`execPath=${execPath}\`)`], + ['Node env overrides', '**INFO**', `\`NODE_OPTIONS=${nodeOptions}\`, \`NODE_EXTRA_CA_CERTS=${extraCa}\`, \`NODE_USE_SYSTEM_CA=${useSystemCa}\``], + ['Node rejects test endpoint', `**${tlsResult.status}**`, tlsResult.detail], + ]; + + await core.summary + .addHeading(heading, 3) + .addTable([ + [{ data: 'Check', header: true }, { data: 'Status', header: true }, { data: 'Details', header: true }], + ...rows, + ]) + .write(); + + if (gating && (!versionOk || tlsResult.status === 'FAIL')) { + core.setFailed(`Node trust probe failed (version=${version}, endpoint=${tlsResult.status})`); + } +}; diff --git a/ci/tools/tests/test_compute_ci_plan.py b/ci/tools/tests/test_compute_ci_plan.py index e1844384d54..b87326420ee 100644 --- a/ci/tools/tests/test_compute_ci_plan.py +++ b/ci/tools/tests/test_compute_ci_plan.py @@ -121,6 +121,10 @@ def test_ignored_paths_select_no_work(self) -> None: "diagram.svg", ".github/labeler.yml", ".github/ISSUE_TEMPLATE/bug.yml", + ".github/workflows/runner-trust-probe.yml", + "ci/tools/runner_trust_probe.sh", + "ci/tools/runner_trust_probe.ps1", + "ci/tools/runner_trust_probe_node.js", ): with self.subTest(path=path): plan = plan_for(path)