diff --git a/.github/actions/install-ci-dependencies/action.yml b/.github/actions/install-ci-dependencies/action.yml new file mode 100644 index 000000000000..d6ef6bc43d91 --- /dev/null +++ b/.github/actions/install-ci-dependencies/action.yml @@ -0,0 +1,94 @@ +name: Install Azure Linux CI dependencies +description: Restore and populate the Azure Linux package cache + +runs: + using: composite + steps: + - name: Configure package cache + id: configure + shell: bash + run: | + set -euo pipefail + + source /etc/os-release + case "$VERSION_ID" in + 3.0) + package_manager=tdnf + setup_script=scripts/setup-ci.sh + cache_dir=/github/home/.ccf-package-cache/tdnf + config_file=/etc/tdnf/tdnf.conf + cache_option=cachedir + keepcache_value=1 + ;; + 4.0) + package_manager=dnf + setup_script=scripts/setup-ci-al4.sh + cache_dir=/github/home/.ccf-package-cache/dnf + config_file=/etc/dnf/dnf.conf + cache_option=system_cachedir + keepcache_value=True + ;; + *) + echo "Unsupported Azure Linux version: $VERSION_ID" >&2 + exit 1 + ;; + esac + + for attempt in 1 2 3; do + if "$package_manager" -y install tar zstd; then + break + fi + if (( attempt == 3 )); then + echo "Failed to install package cache tools" >&2 + exit 1 + fi + if (( attempt == 1 )); then + sleep 5 + else + sleep 30 + fi + done + + set_config_option() { + local name=$1 + local value=$2 + + if grep -q "^${name}=" "$config_file"; then + sed -i "s|^${name}=.*|${name}=${value}|" "$config_file" + else + sed -i "/^\[main\]/a ${name}=${value}" "$config_file" + fi + } + + set_config_option "$cache_option" "$cache_dir" + set_config_option keepcache "$keepcache_value" + mkdir -p "$cache_dir" + + day_of_week=$(date -u +%u) + days_since_sunday=$((day_of_week % 7)) + cache_week=$(date -u -d "$days_since_sunday days ago" +%Y-%m-%d) + + echo "cache-dir=$cache_dir" >> "$GITHUB_OUTPUT" + echo "cache-week=$cache_week" >> "$GITHUB_OUTPUT" + echo "package-manager=$package_manager" >> "$GITHUB_OUTPUT" + echo "setup-script=$setup_script" >> "$GITHUB_OUTPUT" + + - name: Restore package cache + continue-on-error: true + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.configure.outputs.cache-dir }} + key: azure-linux-${{ steps.configure.outputs.package-manager }}-rpms-${{ runner.arch }}-${{ hashFiles('scripts/setup-ci*.sh') }}-${{ steps.configure.outputs.cache-week }} + restore-keys: | + azure-linux-${{ steps.configure.outputs.package-manager }}-rpms-${{ runner.arch }}-${{ hashFiles('scripts/setup-ci*.sh') }}- + azure-linux-${{ steps.configure.outputs.package-manager }}-rpms-${{ runner.arch }}- + + - name: Install dependencies + shell: bash + env: + PACKAGE_MANAGER: ${{ steps.configure.outputs.package-manager }} + SETUP_SCRIPT: ${{ steps.configure.outputs.setup-script }} + run: | + set -euo pipefail + "$PACKAGE_MANAGER" clean metadata + bash "$SETUP_SCRIPT" diff --git a/.github/workflows/README.md b/.github/workflows/README.md index b42293145232..de28b876a813 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,5 +1,13 @@ Documents the various GitHub Actions workflows, the role they fulfill and 3rd party (i.e. outside of https://github.com/actions/) dependencies if any. +# Shared actions + +## Azure Linux CI dependencies + +The local composite action in `.github/actions/install-ci-dependencies/action.yml` installs Azure Linux 3 and 4 CI dependencies and caches downloaded RPMs. Its cache key separates package managers and runner architectures, includes a hash of `scripts/setup-ci*.sh`, and includes the date of the most recent Sunday at midnight UTC. The weekly date makes GitHub Actions create a refreshed immutable cache each week. + +At a weekly rollover, restore keys first reuse the latest cache for the same dependency scripts and then fall back to any cache for the same package manager and architecture. The package manager refreshes repository metadata and downloads only missing or updated RPMs. `actions/cache` saves the populated directory automatically after a successful job when the exact weekly key was not restored. + # Maintained ## Bencher diff --git a/.github/workflows/bencher-ab.yml b/.github/workflows/bencher-ab.yml index 97083690441d..7dd3f8c336be 100644 --- a/.github/workflows/bencher-ab.yml +++ b/.github/workflows/bencher-ab.yml @@ -46,7 +46,7 @@ jobs: fetch-depth: 0 - name: Install dependencies - run: ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: Confirm platform run: python3 tests/infra/platform_detection.py virtual diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index 4f2c91f2231d..e6a6c6c5c253 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -42,10 +42,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Machine info" shell: bash diff --git a/.github/workflows/ci-al4.yml b/.github/workflows/ci-al4.yml index 3141356007dd..21cf4ae9461e 100644 --- a/.github/workflows/ci-al4.yml +++ b/.github/workflows/ci-al4.yml @@ -41,10 +41,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci-al4.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build Debug" run: | @@ -114,10 +111,7 @@ jobs: shell: bash - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci-al4.sh + uses: ./.github/actions/install-ci-dependencies - name: "Confirm running on Virtual" run: | @@ -184,10 +178,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci-al4.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build Debug" run: | diff --git a/.github/workflows/ci-verification.yml b/.github/workflows/ci-verification.yml index 240b624a6810..3a4cb3856d92 100644 --- a/.github/workflows/ci-verification.yml +++ b/.github/workflows/ci-verification.yml @@ -203,10 +203,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: Install TLC dependencies run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8d5b41b83b9..7b93f6d06eeb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,10 +50,12 @@ jobs: fetch-depth: 0 - name: "Install dependencies" + uses: ./.github/actions/install-ci-dependencies + + - name: "Install development dependencies" shell: bash run: | set -ex - ./scripts/setup-ci.sh ./scripts/setup-dev.sh - name: "Run Checks and Build Debug with clang-tidy" @@ -144,10 +146,7 @@ jobs: shell: bash - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Confirm running on Virtual" run: | @@ -215,10 +214,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build Debug" run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index cd03e90724b1..c2d12a1f75ea 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -70,10 +70,7 @@ jobs: queries: security-extended - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Run CMake" run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fb6ed5f4fc6e..e0019f207e50 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -44,10 +44,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build Debug with coverage" run: | diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index d1259b322a0b..abc5c85a803b 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -28,6 +28,9 @@ jobs: with: fetch-depth: 0 + - name: Install dependencies + uses: ./.github/actions/install-ci-dependencies + - name: Setup Pages id: pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 @@ -35,7 +38,6 @@ jobs: - name: Build Documentation run: | set -x - ./scripts/setup-ci.sh python3 -m venv env source env/bin/activate pip install -U pip diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 9fdeb448f264..99bb275a1f53 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -48,10 +48,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build Debug (Long Test)" run: | @@ -109,10 +106,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build Debug (Long Test)" run: | @@ -173,10 +167,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build" run: | @@ -246,10 +237,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build" run: | @@ -311,10 +299,7 @@ jobs: fetch-depth: 0 - name: "Install dependencies" - shell: bash - run: | - set -ex - ./scripts/setup-ci.sh + uses: ./.github/actions/install-ci-dependencies - name: "Build" run: | diff --git a/scripts/setup-ci.sh b/scripts/setup-ci.sh index 3f32d60846e6..ff4da504e8a1 100755 --- a/scripts/setup-ci.sh +++ b/scripts/setup-ci.sh @@ -6,8 +6,11 @@ set -exo pipefail H2SPEC_VERSION="v2.6.0" -export SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH:-$(date +%s)} -echo "Using SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" +TDNF_OPTIONS=(-y) +if [[ -n ${SOURCE_DATE_EPOCH:-} ]]; then + echo "Using SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" + TDNF_OPTIONS+=("--snapshottime=$SOURCE_DATE_EPOCH") +fi retry() { local description=$1 @@ -44,14 +47,14 @@ retry() { install_source_control() { # Source control - tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ + tdnf "${TDNF_OPTIONS[@]}" install \ git \ ca-certificates } install_build_dependencies() { # To build CCF - tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ + tdnf "${TDNF_OPTIONS[@]}" install \ build-essential \ clang \ cmake \ @@ -85,7 +88,7 @@ install_test_dependencies() { iptables strace ) - tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install "${packages[@]}" && + tdnf "${TDNF_OPTIONS[@]}" install "${packages[@]}" && gem install cddl } @@ -106,7 +109,7 @@ install_node() { # constraint pins the major version (failing rather than silently selecting # an older nodejs); `nodejs-npm` provides npm and depends on that same # `nodejs`, so it follows the selected version. - tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install \ + tdnf "${TDNF_OPTIONS[@]}" install \ "nodejs >= 24" \ nodejs-npm } @@ -118,7 +121,7 @@ install_packaging_and_python() { # For end to end tests and scripts python3-pip ) - tdnf --snapshottime=$SOURCE_DATE_EPOCH -y install "${packages[@]}" && + tdnf "${TDNF_OPTIONS[@]}" install "${packages[@]}" && pip install uv==0.11.19 }