WIP: [DO NOT MERGE] introduce libcugraph wheels

.github/workflows/build.yaml

@@ -67,7 +67,29 @@ jobs:
       node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
       sha: ${{ inputs.sha }}
+  wheel-build-libcugraph:
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      script: ci/build_wheel_libcugraph.sh
+      node_type: cpu32
+  wheel-publish-libcugraph:
+    needs: wheel-publish-libcugraph
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      build_type: ${{ inputs.build_type || 'branch' }}
+      branch: ${{ inputs.branch }}
+      sha: ${{ inputs.sha }}
+      date: ${{ inputs.date }}
+      package-name: libcugraph
+      package-type: cpp
   wheel-build-pylibcugraph:
+    needs: wheel-publish-libcugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
@@ -76,7 +98,6 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       script: ci/build_wheel_pylibcugraph.sh
-      node_type: cpu32
   wheel-publish-pylibcugraph:
     needs: wheel-build-pylibcugraph
     secrets: inherit
@@ -87,6 +108,7 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: pylibcugraph
+      package-type: python
   wheel-build-cugraph:
     needs: wheel-publish-pylibcugraph
     secrets: inherit
@@ -107,3 +129,4 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cugraph
+      package-type: cpp

.github/workflows/pr.yaml

@@ -13,51 +13,52 @@ jobs:
   # Please keep pr-builder as the top job here
   pr-builder:
     needs:
-      - check-nightly-ci
+      #- check-nightly-ci
       - changed-files
       - checks
       - conda-cpp-build
       - conda-cpp-tests
       - conda-cpp-checks
-      - conda-notebook-tests
+      # - conda-notebook-tests
       - conda-python-build
       - conda-python-tests
-      - docs-build
+      # - docs-build
+      - wheel-build-libcugraph
       - wheel-build-pylibcugraph
       - wheel-tests-pylibcugraph
       - wheel-build-cugraph
       - wheel-tests-cugraph
-      - telemetry-setup
-      - devcontainer
+      # - telemetry-setup
+      # - devcontainer
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: always()
     with:
       needs: ${{ toJSON(needs) }}
-  telemetry-setup:
-    runs-on: ubuntu-latest
-    continue-on-error: true
-    env:
-        OTEL_SERVICE_NAME: "pr-cugraph"
-    steps:
-      - name: Telemetry setup
-        if: ${{ vars.TELEMETRY_ENABLED == 'true' }}
-        uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main
-  check-nightly-ci:
-    # Switch to ubuntu-latest once it defaults to a version of Ubuntu that
-    # provides at least Python 3.11 (see
-    # https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat)
-    runs-on: ubuntu-24.04
-    env:
-      RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - name: Check if nightly CI is passing
-        uses: rapidsai/shared-actions/check_nightly_success/dispatch@main
-        with:
-          repo: cugraph
+  # telemetry-setup:
+  #   runs-on: ubuntu-latest
+  #   continue-on-error: true
+  #   env:
+  #       OTEL_SERVICE_NAME: "pr-cugraph"
+  #   steps:
+  #     - name: Telemetry setup
+  #       if: ${{ vars.TELEMETRY_ENABLED == 'true' }}
+  #       uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main
+  # check-nightly-ci:
+  #   # Switch to ubuntu-latest once it defaults to a version of Ubuntu that
+  #   # provides at least Python 3.11 (see
+  #   # https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat)
+  #   runs-on: ubuntu-24.04
+  #   env:
+  #     RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  #   steps:
+  #     - name: Check if nightly CI is passing
+  #       uses: rapidsai/shared-actions/check_nightly_success/dispatch@main
+  #       with:
+  #         repo: cugraph
   changed-files:
     secrets: inherit
-    needs: telemetry-setup
+    # needs: telemetry-setup
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       files_yaml: |
@@ -88,7 +89,7 @@ jobs:
           - '!notebooks/**'
   checks:
     secrets: inherit
-    needs: telemetry-setup
+    # needs: telemetry-setup
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       enable_check_generated_files: false
@@ -128,35 +129,43 @@ jobs:
     if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
     with:
       build_type: pull-request
-  conda-notebook-tests:
-    needs: [conda-python-build, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
-    with:
-      build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
-      run_script: "ci/test_notebooks.sh"
-  docs-build:
-    needs: conda-python-build
+  # conda-notebook-tests:
+  #   needs: [conda-python-build, changed-files]
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
+  #     run_script: "ci/test_notebooks.sh"
+  # docs-build:
+  #   needs: conda-python-build
+  #   secrets: inherit
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     build_type: pull-request
+  #     node_type: "gpu-v100-latest-1"
+  #     arch: "amd64"
+  #     container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
+  #     run_script: "ci/build_docs.sh"
+  wheel-build-libcugraph:
+    # needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job[email protected]
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build[email protected]
     with:
+      # build for every combination of arch and CUDA version, but only for the latest Python
+      matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       build_type: pull-request
-      node_type: "gpu-v100-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10"
-      run_script: "ci/build_docs.sh"
+      script: ci/build_wheel_libcugraph.sh
   wheel-build-pylibcugraph:
-    needs: checks
+    needs: wheel-build-libcugraph
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
       script: ci/build_wheel_pylibcugraph.sh
-      node_type: cpu32
   wheel-tests-pylibcugraph:
     needs: [wheel-build-pylibcugraph, changed-files]
     secrets: inherit
@@ -176,28 +185,30 @@ jobs:
     needs: [wheel-build-cugraph, changed-files]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
+    # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
     with:
       build_type: pull-request
       script: ci/test_wheel_cugraph.sh
-  devcontainer:
-    secrets: inherit
-    needs: telemetry-setup
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      arch: '["amd64"]'
-      cuda: '["12.5"]'
-      node_type: cpu32
-      build_command: |
-        sccache -z;
-        build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON;
-        sccache -s;
-  telemetry-summarize:
-    # This job must use a self-hosted runner to record telemetry traces.
-    runs-on: linux-amd64-cpu4
-    needs: pr-builder
-    if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }}
-    continue-on-error: true
-    steps:
-      - name: Telemetry summarize
-        uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main
+      # This selects "ARCH=amd64 + the latest supported Python + CUDA".
+      matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.]
+  # devcontainer:
+  #   secrets: inherit
+  #   # needs: telemetry-setup
+  #   uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+  #   with:
+  #     arch: '["amd64"]'
+  #     cuda: '["12.5"]'
+  #     node_type: cpu32
+  #     build_command: |
+  #       sccache -z;
+  #       build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON;
+  #       sccache -s;
+  # telemetry-summarize:
+  #   # This job must use a self-hosted runner to record telemetry traces.
+  #   runs-on: linux-amd64-cpu4
+  #   needs: pr-builder
+  #   if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }}
+  #   continue-on-error: true
+  #   steps:
+  #     - name: Telemetry summarize
+  #       uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main

build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2019-2024, NVIDIA CORPORATION.
+# Copyright (c) 2019-2025, NVIDIA CORPORATION.
 
 # cugraph build script
 
@@ -175,11 +175,6 @@ SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}"
 # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS
 SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g')
 
-# Append `-DFIND_CUGRAPH_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option.
-if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUGRAPH_CPP"* ]]; then
-    SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUGRAPH_CPP=ON"
-fi
-
 # If clean or uninstall targets given, run them prior to any other steps
 if hasArg uninstall; then
     if [[ "$INSTALL_PREFIX" != "" ]]; then

ci/build_cpp.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -9,6 +9,9 @@ source rapids-configure-sccache
 
 source rapids-date-string
 
+# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged
+source ci/use_conda_packages_from_prs.sh
+
 export CMAKE_GENERATOR=Ninja
 
 rapids-print-env

ci/build_docs.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -10,6 +10,9 @@ export RAPIDS_VERSION="$(rapids-version)"
 export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
 export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR"
 
+# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged
+source ci/use_conda_packages_from_prs.sh
+
 rapids-dependency-file-generator \
   --output conda \
   --file-key docs \

ci/build_python.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -9,6 +9,9 @@ source rapids-configure-sccache
 
 source rapids-date-string
 
+# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged
+source ci/use_conda_packages_from_prs.sh
+
 export CMAKE_GENERATOR=Ninja
 
 rapids-print-env

ci/build_wheel.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
-# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
 package_name=$1
 package_dir=$2
+package_type=$3
 
 source rapids-configure-sccache
 source rapids-date-string
@@ -29,9 +30,15 @@ python -m pip wheel \
 
 sccache --show-adv-stats
 
+# TODO(jameslamb): move all the auditwheel stuff into individual build_wheel_{project}.sh like cudf does?
+
+EXCLUDE_ARGS=(
+  --exclude "libraft.so"
+)
+
 case "${RAPIDS_CUDA_VERSION}" in
     12.*)
-        EXCLUDE_ARGS=(
+        EXCLUDE_ARGS+=(
             --exclude "libcublas.so.12"
             --exclude "libcublasLt.so.12"
             --exclude "libcurand.so.10"
@@ -40,11 +47,23 @@ case "${RAPIDS_CUDA_VERSION}" in
             --exclude "libnvJitLink.so.12"
         )
     ;;
-    11.*)
-        EXCLUDE_ARGS=()
-    ;;
+esac
+
+case "${package_dir}" in
+  python/pylibcugraph)
+    EXCLUDE_ARGS+=(
+      --exclude "libcugraph_c.so"
+      --exclude "libcugraph.so"
+    )
+  ;;
+  python/cugraph)
+    EXCLUDE_ARGS+=(
+      --exclude "libcugraph_c.so"
+      --exclude "libcugraph.so"
+    )
+  ;;
 esac
 
 mkdir -p final_dist
 python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/*
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist