diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 357d0d9bca..a206e66c42 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,29 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-libcugraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_libcugraph.sh + node_type: cpu32 + wheel-publish-libcugraph: + needs: wheel-publish-libcugraph + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-25.02 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcugraph + package-type: cpp wheel-build-pylibcugraph: + needs: wheel-publish-libcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: @@ -76,7 +98,6 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} script: ci/build_wheel_pylibcugraph.sh - node_type: cpu32 wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -87,6 +108,7 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibcugraph + package-type: python wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit @@ -107,3 +129,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cugraph + package-type: cpp diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c167da995c..4ff79f0609 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,51 +13,52 @@ jobs: # Please keep pr-builder as the top job here pr-builder: needs: - - check-nightly-ci + #- check-nightly-ci - changed-files - checks - conda-cpp-build - conda-cpp-tests - conda-cpp-checks - - conda-notebook-tests + # - conda-notebook-tests - conda-python-build - conda-python-tests - - docs-build + # - docs-build + - wheel-build-libcugraph - wheel-build-pylibcugraph - wheel-tests-pylibcugraph - wheel-build-cugraph - wheel-tests-cugraph - - telemetry-setup - - devcontainer + # - telemetry-setup + # - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 if: always() with: needs: ${{ toJSON(needs) }} - telemetry-setup: - runs-on: ubuntu-latest - continue-on-error: true - env: - OTEL_SERVICE_NAME: "pr-cugraph" - steps: - - name: Telemetry setup - if: ${{ vars.TELEMETRY_ENABLED == 'true' }} - uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main - check-nightly-ci: - # Switch to ubuntu-latest once it defaults to a version of Ubuntu that - # provides at least Python 3.11 (see - # https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat) - runs-on: ubuntu-24.04 - env: - RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Check if nightly CI is passing - uses: rapidsai/shared-actions/check_nightly_success/dispatch@main - with: - repo: cugraph + # telemetry-setup: + # runs-on: ubuntu-latest + # continue-on-error: true + # env: + # OTEL_SERVICE_NAME: "pr-cugraph" + # steps: + # - name: Telemetry setup + # if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + # uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main + # check-nightly-ci: + # # Switch to ubuntu-latest once it defaults to a version of Ubuntu that + # # provides at least Python 3.11 (see + # # https://docs.python.org/3/library/datetime.html#datetime.date.fromisoformat) + # runs-on: ubuntu-24.04 + # env: + # RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # steps: + # - name: Check if nightly CI is passing + # uses: rapidsai/shared-actions/check_nightly_success/dispatch@main + # with: + # repo: cugraph changed-files: secrets: inherit - needs: telemetry-setup + # needs: telemetry-setup uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.02 with: files_yaml: | @@ -88,7 +89,7 @@ jobs: - '!notebooks/**' checks: secrets: inherit - needs: telemetry-setup + # needs: telemetry-setup uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.02 with: enable_check_generated_files: false @@ -128,35 +129,43 @@ jobs: if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request - conda-notebook-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build + # conda-notebook-tests: + # needs: [conda-python-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + # run_script: "ci/test_notebooks.sh" + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + # run_script: "ci/build_docs.sh" + wheel-build-libcugraph: + # needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.02 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/build_docs.sh" + script: ci/build_wheel_libcugraph.sh wheel-build-pylibcugraph: - needs: checks + needs: wheel-build-libcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request script: ci/build_wheel_pylibcugraph.sh - node_type: cpu32 wheel-tests-pylibcugraph: needs: [wheel-build-pylibcugraph, changed-files] secrets: inherit @@ -176,28 +185,30 @@ jobs: needs: [wheel-build-cugraph, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_cugraph.sh - devcontainer: - secrets: inherit - needs: telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - node_type: cpu32 - build_command: | - sccache -z; - build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; - sccache -s; - telemetry-summarize: - # This job must use a self-hosted runner to record telemetry traces. - runs-on: linux-amd64-cpu4 - needs: pr-builder - if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} - continue-on-error: true - steps: - - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] + # devcontainer: + # secrets: inherit + # # needs: telemetry-setup + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-25.02 + # with: + # arch: '["amd64"]' + # cuda: '["12.5"]' + # node_type: cpu32 + # build_command: | + # sccache -z; + # build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; + # sccache -s; + # telemetry-summarize: + # # This job must use a self-hosted runner to record telemetry traces. + # runs-on: linux-amd64-cpu4 + # needs: pr-builder + # if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} + # continue-on-error: true + # steps: + # - name: Telemetry summarize + # uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main diff --git a/build.sh b/build.sh index 0282948659..ebdc6ed5c5 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2019-2024, NVIDIA CORPORATION. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. # cugraph build script @@ -175,11 +175,6 @@ SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" # Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') -# Append `-DFIND_CUGRAPH_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_CUGRAPH_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DFIND_CUGRAPH_CPP=ON" -fi - # If clean or uninstall targets given, run them prior to any other steps if hasArg uninstall; then if [[ "$INSTALL_PREFIX" != "" ]]; then diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 93e5c680d0..1903cca673 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -9,6 +9,9 @@ source rapids-configure-sccache source rapids-date-string +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + export CMAKE_GENERATOR=Ninja rapids-print-env diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 938a2b5554..e71d3345b8 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -10,6 +10,9 @@ export RAPIDS_VERSION="$(rapids-version)" export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)" export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR" +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + rapids-dependency-file-generator \ --output conda \ --file-key docs \ diff --git a/ci/build_python.sh b/ci/build_python.sh index eab41f63da..e776ad90c9 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail @@ -9,6 +9,9 @@ source rapids-configure-sccache source rapids-date-string +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ci/use_conda_packages_from_prs.sh + export CMAKE_GENERATOR=Ninja rapids-print-env diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9a77e6b302..28fc97b34e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -1,10 +1,11 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail package_name=$1 package_dir=$2 +package_type=$3 source rapids-configure-sccache source rapids-date-string @@ -29,9 +30,15 @@ python -m pip wheel \ sccache --show-adv-stats +# TODO(jameslamb): move all the auditwheel stuff into individual build_wheel_{project}.sh like cudf does? + +EXCLUDE_ARGS=( + --exclude "libraft.so" +) + case "${RAPIDS_CUDA_VERSION}" in 12.*) - EXCLUDE_ARGS=( + EXCLUDE_ARGS+=( --exclude "libcublas.so.12" --exclude "libcublasLt.so.12" --exclude "libcurand.so.10" @@ -40,11 +47,23 @@ case "${RAPIDS_CUDA_VERSION}" in --exclude "libnvJitLink.so.12" ) ;; - 11.*) - EXCLUDE_ARGS=() - ;; +esac + +case "${package_dir}" in + python/pylibcugraph) + EXCLUDE_ARGS+=( + --exclude "libcugraph_c.so" + --exclude "libcugraph.so" + ) + ;; + python/cugraph) + EXCLUDE_ARGS+=( + --exclude "libcugraph_c.so" + --exclude "libcugraph.so" + ) + ;; esac mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" final_dist diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 11ab9acc58..5c4741d772 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail @@ -7,31 +7,24 @@ package_dir="python/cugraph" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibcugraph wheel built in the previous step and make it +# Download the libcugraph and pylibcugraph wheels built in the previous step and make them # available for pip to find. -# -# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run -# -# using env variable PIP_CONSTRAINT is necessary to ensure the constraints -# are used when creating the isolated build environment -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist) - -echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt +LIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcugraph_dist) +PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) + +# TODO(jameslamb): remove this when https://github.com/rapidsai/raft/pull/2531 is merged +source ./ci/use_wheels_from_prs.sh + +cat >> ./constraints.txt <> ./constraints.txt < ./constraints.txt <=0.0.0a0 + - &libraft_unsuffixed libraft==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libraft-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libraft-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libraft_unsuffixed]} depends_on_librmm: common: - output_types: conda packages: - - librmm==25.2.*,>=0.0.0a0 + - &librmm_unsuffixed librmm==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} depends_on_rmm: common: @@ -721,6 +793,31 @@ dependencies: - raft-dask-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*raft_dask_unsuffixed]} + depends_on_libcugraph: + common: + - output_types: conda + packages: + - &libcugraph_unsuffixed libcugraph==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libcugraph-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libcugraph-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*libcugraph_unsuffixed]} + depends_on_pylibcugraph: common: - output_types: conda diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 117f7cf514..68978c0865 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,57 +27,10 @@ project( LANGUAGES CXX CUDA ) -################################################################################ -# - User Options -------------------------------------------------------------- -option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files" - OFF -) -option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) - - -# If the user requested it, we attempt to find CUGRAPH. -if(FIND_CUGRAPH_CPP) - find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -else() - set(cugraph_FOUND OFF) -endif() +find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) -if(NOT cugraph_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_TESTS OFF) - set(BUILD_CUGRAPH_MG_TESTS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_LIB ON) - - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - - add_subdirectory(../../cpp cugraph-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set(rpaths - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/cufft/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) - endif() - - set(cython_lib_dir cugraph) - install(TARGETS cugraph DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(cugraph/components) @@ -89,7 +42,3 @@ add_subdirectory(cugraph/linear_assignment) add_subdirectory(cugraph/structure) add_subdirectory(cugraph/tree) add_subdirectory(cugraph/utilities) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cugraph PATHS "${cython_lib_dir}") -endif() diff --git a/python/cugraph/cugraph/__init__.py b/python/cugraph/cugraph/__init__.py index ada1fec74c..0058ab4a09 100644 --- a/python/cugraph/cugraph/__init__.py +++ b/python/cugraph/cugraph/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2024, NVIDIA CORPORATION. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,6 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# If libcugraph was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcugraph +except ModuleNotFoundError: + pass +else: + libcugraph.load_library() + del libcugraph + from cugraph.community import ( ecg, induced_subgraph, diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 8fa3a0938e..ba22cb9f84 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "dask-cuda==25.2.*,>=0.0.0a0", "dask-cudf==25.2.*,>=0.0.0a0", "fsspec[http]>=0.6.0", + "libcugraph==25.2.*,>=0.0.0a0", "numba>=0.57", "numpy>=1.23,<3.0a0", "pylibcugraph==25.2.*,>=0.0.0a0", @@ -83,6 +84,9 @@ regex = "(?P.*)" build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4,!=3.30.0", + "libcugraph==25.2.*,>=0.0.0a0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibcugraph==25.2.*,>=0.0.0a0", "pylibraft==25.2.*,>=0.0.0a0", @@ -93,6 +97,8 @@ matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ - # NOTE: size threshold is managed via CLI args in CI scripts "distro-too-large-compressed", ] + +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M' diff --git a/python/libcugraph/CMakeLists.txt b/python/libcugraph/CMakeLists.txt new file mode 100644 index 0000000000..69108cb98d --- /dev/null +++ b/python/libcugraph/CMakeLists.txt @@ -0,0 +1,68 @@ +# ============================================================================= +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libcugraph-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) + +# Check if cugraph is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of Python packages that need libcugraph. +find_package(cugraph "${RAPIDS_VERSION}") + +if(cugraph_FOUND) + return() +endif() + +unset(cugraph_FOUND) + +# --- CUDA -- # +find_package(CUDAToolkit REQUIRED) +set(CUDA_STATIC_RUNTIME ON) +set(CUDA_STATIC_MATH_LIBRARIES ON) +if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) +elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") +endif() + +# --- cugraph ---# +set(BUILD_TESTS OFF) +set(BUILD_CUGRAPH_MG_TESTS OFF) +set(CMAKE_CUDA_LINEINFO OFF) + +# --- raft -- # +set(USE_RAFT_STATIC OFF) +set(CUGRAPH_COMPILE_RAFT_LIB ON) + +add_subdirectory(../../cpp cugraph-cpp) + +if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set(rpaths + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/cufft/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) + set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) +endif() diff --git a/python/libcugraph/LICENSE b/python/libcugraph/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libcugraph/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcugraph/README.md b/python/libcugraph/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libcugraph/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcugraph/libcugraph/VERSION b/python/libcugraph/libcugraph/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libcugraph/libcugraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libcugraph/libcugraph/__init__.py b/python/libcugraph/libcugraph/__init__.py new file mode 100644 index 0000000000..ba9b02354b --- /dev/null +++ b/python/libcugraph/libcugraph/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcugraph._version import __git_commit__, __version__ +from libcugraph.load import load_library diff --git a/python/libcugraph/libcugraph/_version.py b/python/libcugraph/libcugraph/_version.py new file mode 100644 index 0000000000..da66c0d576 --- /dev/null +++ b/python/libcugraph/libcugraph/_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcugraph/libcugraph/load.py b/python/libcugraph/libcugraph/load.py new file mode 100644 index 0000000000..0345c8e7bb --- /dev/null +++ b/python/libcugraph/libcugraph/load.py @@ -0,0 +1,98 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libcugraph that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile(lib := os.path.join(os.path.dirname(__file__), "lib64", soname)): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libcugraph.so and its dependencies""" + try: + # libraft must be loaded before libcugraph because libcugraph + # references its symbols + import libraft + + libraft.load_library() + except ModuleNotFoundError: + # 'libcugraph' has a runtime dependency on 'libraft'. However, + # that dependency might be satisfied by the 'libraft' conda package + # (which does not have any Python modules), instead of the + # 'libraft' wheel. + # + # In that situation, assume that 'libraft.so' is in a place where + # the loader can find it. + pass + + prefer_system_installation = ( + os.getenv("RAPIDS_LIBCUGRAPH_PREFER_SYSTEM_LIBRARY", "false").lower() != "false" + ) + + libs_to_return = [] + for soname in ["libcugraph.so", "libcugraph_c.so"]: + libcugraph_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libcugraph_lib = _load_system_installation(soname) + except OSError: + libcugraph_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + try: + libcugraph_lib = _load_wheel_installation(soname) + if libcugraph_lib is None: + libcugraph_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass + if libcugraph_lib: + libs_to_return.append(libcugraph_lib) + + # The caller almost never needs to do anything with these libraries, but no + # harm in offering the option since these objects at least provide handles + # to inspect where libcugraph was loaded from. + + return libs_to_return diff --git a/python/libcugraph/pyproject.toml b/python/libcugraph/pyproject.toml new file mode 100644 index 0000000000..c79e4f7904 --- /dev/null +++ b/python/libcugraph/pyproject.toml @@ -0,0 +1,87 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.1,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcugraph" +dynamic = ["version"] +description = "cuGraph - RAPIDS GPU Graph Analytics (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.10" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "libraft==25.2.*,>=0.0.0a0", + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/cugraph" + +[project.entry-points."cmake.prefix"] +libcugraph = "libcugraph" + +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '1.3G' + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcugraph"] +wheel.install-dir = "libcugraph" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcugraph/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 22d788ff25..b9576c1d72 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -27,60 +27,10 @@ project( LANGUAGES CXX CUDA ) -################################################################################ -# - User Options -------------------------------------------------------------- -option(FIND_CUGRAPH_CPP "Search for existing CUGRAPH C++ installations before defaulting to local files" - OFF -) -option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) - -# If the user requested it we attempt to find CUGRAPH. -if(FIND_CUGRAPH_CPP) - find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) -else() - set(cugraph_FOUND OFF) -endif() +find_package(cugraph "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) -if (NOT cugraph_FOUND) - find_package(CUDAToolkit REQUIRED) - - set(BUILD_TESTS OFF) - set(BUILD_CUGRAPH_MG_TESTS OFF) - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_LIB ON) - - if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) - set(CUDA_STATIC_MATH_LIBRARIES OFF) - elseif(USE_CUDA_MATH_WHEELS) - message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") - endif() - - add_subdirectory(../../cpp cugraph-cpp EXCLUDE_FROM_ALL) - - if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) - set(rpaths - "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/curand/lib" - "$ORIGIN/../nvidia/cusolver/lib" - "$ORIGIN/../nvidia/cusparse/lib" - "$ORIGIN/../nvidia/nvjitlink/lib" - ) - set_property(TARGET cugraph PROPERTY INSTALL_RPATH ${rpaths} APPEND) - endif() - - set(cython_lib_dir pylibcugraph) - install(TARGETS cugraph DESTINATION ${cython_lib_dir}) - install(TARGETS cugraph_c DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(pylibcugraph) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET cugraph PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 9047144c13..f9561d961c 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2024, NVIDIA CORPORATION. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,6 +11,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# If libcugraph was installed as a wheel, we must request it to load the library +# symbols. Otherwise, we assume that the library was installed in a system path that ld +# can find. +try: + import libcugraph +except ModuleNotFoundError: + pass +else: + libcugraph.load_library() + del libcugraph + from pylibcugraph.components._connectivity import ( strongly_connected_components, ) diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 72a5e19c70..4cbb4a3214 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -3,7 +3,6 @@ [build-system] requires = [ - "cython>=3.0.0", "rapids-build-backend>=0.3.1,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.10.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -23,10 +22,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "nvidia-cublas", - "nvidia-curand", - "nvidia-cusolver", - "nvidia-cusparse", + "libcugraph==25.2.*,>=0.0.0a0", "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -73,14 +69,20 @@ build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4,!=3.30.0", + "cython>=3.0.0", + "libcugraph==25.2.*,>=0.0.0a0", + "libraft==25.2.*,>=0.0.0a0", + "librmm==25.2.*,>=0.0.0a0", "ninja", "pylibraft==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" +matrix-entry = "cuda_suffixed=true" [tool.pydistcheck] select = [ - # NOTE: size threshold is managed via CLI args in CI scripts "distro-too-large-compressed", ] + +# PyPI limit is 100 MiB, fail CI before we get too close to that +max_allowed_size_compressed = '75M'