From 47bac702c005c077b4df082d120cafed608913d5 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 13 Jan 2025 21:05:40 -0600 Subject: [PATCH] declare cuda-python dependency for wheels, other small packaging changes (#6217) Follow-up to #6190. Proposes some miscellaneous packaging cleanup: * declares `cuml-cu{11,12}` wheels' runtime dependency on `cuda-python` - *as a result of stuff like this: https://github.com/rapidsai/cuml/blob/bfd2e220d3adf5d8c6b76dc90e3d1275054f32d5/python/cuml/cuml/svm/linear.pyx#L40-L43* *~ adds `raft_log.txt` to `.gitignore`~ * adds CMake option `CUML_USE_RAFT_STATIC` - *to provide a default for this: https://github.com/rapidsai/cuml/blob/bfd2e220d3adf5d8c6b76dc90e3d1275054f32d5/cpp/CMakeLists.txt#L600* * defines `BUILD_CAGRA_HNSWLIB OFF` in `get_cuvs.cmake` - *as is done for RAFT: https://github.com/rapidsai/cuml/blob/bfd2e220d3adf5d8c6b76dc90e3d1275054f32d5/cpp/cmake/thirdparty/get_raft.cmake#L58* - *cuML doesn't need the CAGRA stuff from cuVS, as far as I can tell* - *this is `ON` by default in cuVS, so this change saves a bit of build time and size: https://github.com/rapidsai/cuvs/blob/1e548f8c3a773452ce69556f4db72fc712efae02/cpp/CMakeLists.txt#L58* * explicitly passing package type to `rapids-download-wheels-from-s3` in CI scripts ## Notes for Reviewers These changes are useful independently, but will also make the `libcuml` wheels PR (https://github.com/rapidsai/cuml/pull/6199) a bit smaller and easier to review. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cuml/pull/6217 --- .github/workflows/build.yaml | 1 + ci/test_wheel.sh | 7 +++-- cpp/CMakeLists.txt | 2 ++ cpp/cmake/thirdparty/get_cuvs.cmake | 3 +- dependencies.yaml | 46 ++++++++++++++++------------- python/cuml/CMakeLists.txt | 4 +-- python/cuml/pyproject.toml | 1 + 7 files changed, 37 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9a4b614584..945b8e4f72 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -91,3 +91,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cuml + package-type: python diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 047575a2d7..76cfe9a86b 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -1,14 +1,15 @@ #!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuml*.whl)[test] +python -m pip install \ + "$(echo ./dist/cuml*.whl)[test]" RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} mkdir -p "${RAPIDS_TESTS_DIR}" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bfa9cba085..90c0c02cf3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -65,6 +65,7 @@ option(USE_CCACHE "Cache build artifacts with ccache" OFF) option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) option(CUDA_STATIC_MATH_LIBRARIES "Statically link the CUDA math libraries" OFF) option(CUML_USE_CUVS_STATIC "Build and statically link the CUVS library" OFF) +option(CUML_USE_RAFT_STATIC "Build and statically link the RAFT library" OFF) option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF) option(CUML_EXPORT_TREELITE_LINKAGE "Whether to publicly or privately link treelite to libcuml++" OFF) option(CUML_USE_CUMLPRIMS_MG_STATIC "Build and statically link the cumlprims_mg library" OFF) @@ -99,6 +100,7 @@ message(VERBOSE "CUML_CPP: Cache build artifacts with ccache: ${USE_CCACHE}") message(VERBOSE "CUML_CPP: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") message(VERBOSE "CUML_CPP: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}") message(VERBOSE "CUML_CPP: Build and statically link CUVS libraries: ${CUML_USE_CUVS_STATIC}") +message(VERBOSE "CUML_CPP: Build and statically link RAFT library: ${CUML_USE_RAFT_STATIC}") message(VERBOSE "CUML_CPP: Build and statically link Treelite library: ${CUML_USE_TREELITE_STATIC}") set(CUML_ALGORITHMS "ALL" CACHE STRING "Experimental: Choose which algorithms are built into libcuml++.so. Can specify individual algorithms or groups in a semicolon-separated list.") diff --git a/cpp/cmake/thirdparty/get_cuvs.cmake b/cpp/cmake/thirdparty/get_cuvs.cmake index a46879c3e7..a48b4c6b1e 100644 --- a/cpp/cmake/thirdparty/get_cuvs.cmake +++ b/cpp/cmake/thirdparty/get_cuvs.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -55,6 +55,7 @@ function(find_and_configure_cuvs) OPTIONS "BUILD_TESTS OFF" "BUILD_BENCH OFF" + "BUILD_CAGRA_HNSWLIB OFF" "BUILD_MG_ALGOS ${CUVS_BUILD_MG_ALGOS}" ) diff --git a/dependencies.yaml b/dependencies.yaml index 90e6dc5fc2..f35eb74021 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -10,6 +10,7 @@ files: - cuda - cuda_version - depends_on_cudf + - depends_on_cuda_python - depends_on_cupy - depends_on_cuvs - depends_on_dask_cudf @@ -21,8 +22,8 @@ files: - depends_on_raft_dask - depends_on_rmm - docs - - py_build - - py_run + - py_build_cuml + - py_run_cuml - py_version - rapids_build_backend - test_python @@ -90,7 +91,7 @@ files: - depends_on_pylibraft - depends_on_raft_dask - depends_on_rmm - - py_run + - py_run_cuml - py_version - test_notebooks py_build_cuml: @@ -108,12 +109,13 @@ files: key: requires includes: - common_build + - depends_on_cuda_python - depends_on_cuvs - depends_on_libcumlprims - depends_on_libraft_headers - depends_on_pylibraft - depends_on_rmm - - py_build + - py_build_cuml py_run_cuml: output: pyproject pyproject_dir: python/cuml @@ -121,6 +123,7 @@ files: table: project includes: - cuda_wheels + - depends_on_cuda_python - depends_on_cudf - depends_on_cupy - depends_on_cuvs @@ -128,7 +131,7 @@ files: - depends_on_pylibraft - depends_on_raft_dask - depends_on_rmm - - py_run + - py_run_cuml py_test_cuml: output: pyproject pyproject_dir: python/cuml @@ -210,28 +213,14 @@ dependencies: cuda: "12.*" packages: - cuda-nvcc - py_build: + py_build_cuml: common: - output_types: [conda, requirements, pyproject] packages: - &cython cython>=3.0.0 - &treelite treelite==4.3.0 - specific: - - output_types: [conda, requirements, pyproject] - matrices: - - matrix: - cuda: "12.*" - packages: - - cuda-python>=12.6.2,<13.0a0 - - matrix: - cuda: "11.*" - packages: - - cuda-python>=11.8.5,<12.0a0 - - matrix: - packages: - - cuda-python - py_run: + py_run_cuml: common: - output_types: [conda, requirements, pyproject] packages: @@ -469,6 +458,21 @@ dependencies: - *scikit_learn - seaborn - *xgboost + depends_on_cuda_python: + specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + packages: + - cuda-python>=12.6.2,<13.0a0 + - matrix: + cuda: "11.*" + packages: + - cuda-python>=11.8.5,<12.0a0 + - matrix: + packages: + - cuda-python depends_on_cudf: common: - output_types: conda diff --git a/python/cuml/CMakeLists.txt b/python/cuml/CMakeLists.txt index 1dcaecc92f..7bda8d9a64 100644 --- a/python/cuml/CMakeLists.txt +++ b/python/cuml/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -174,7 +174,7 @@ endif() include("${CUML_CPP_SRC}/cmake/modules/ConfigureAlgorithms.cmake") include(cmake/ConfigureCythonAlgorithms.cmake) -if(${CUML_CPU}) +if(CUML_CPU) # libcuml requires metrics built if HDSCAN is built, which is not the case # for cuml-cpu unset(metrics_algo) diff --git a/python/cuml/pyproject.toml b/python/cuml/pyproject.toml index 0bb733ef59..820a690d3c 100644 --- a/python/cuml/pyproject.toml +++ b/python/cuml/pyproject.toml @@ -92,6 +92,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ + "cuda-python", "cudf==25.2.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", "cuvs==25.2.*,>=0.0.0a0",