Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

proprietary_binary now supports cuda toolkit version placeholders #377

Merged
merged 5 commits into from
Feb 27, 2023
Merged

proprietary_binary now supports cuda toolkit version placeholders #377

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a8ff7fa
proprietary_binary now supports cuda toolkit version placeholders
robertmaynard Feb 21, 2023
3d99a6f
Correct copyright year
robertmaynard Feb 22, 2023
6d41047
Simplify how get_proprietary_binary_url find CTK version
robertmaynard Feb 22, 2023
36effbb
Remove unneeded logic found in code review
robertmaynard Feb 22, 2023
a850900
Correct spelling found during code review
robertmaynard Feb 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/packages/rapids_cpm_versions.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ as needed.
Supports the following placeholders:
- ``${rapids-cmake-version}`` will be evaluated to 'major.minor' of the current rapids-cmake cal-ver value.
- ``${version}`` will be evaluated to the contents of the ``version`` field.
- ``${cuda-toolkit-version}`` will be evaluated to 'major.minor' of the current CUDA Toolkit version.
- ``${cuda-toolkit-version-major}`` will be evaluated to 'major' of the current CUDA Toolkit version.

If this field exists in the default package, the value will be ignored when an override file
entry exists for the package. This ensures that the git url or `proprietary_binary` entry in the override will be used.
Expand Down
46 changes: 46 additions & 0 deletions rapids-cmake/cpm/detail/download_proprietary_binary.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#=============================================================================
# Copyright (c) 2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#=============================================================================
include_guard(GLOBAL)

#[=======================================================================[.rst:
download_proprietary_binary
-------------------

.. versionadded:: v23.04.00

Download the associated proprietary binary from the providied URL and make
it part of the project with `FetchContent_MakeAvailable`

#]=======================================================================]
function(rapids_cpm_download_proprietary_binary package_name url)
list(APPEND CMAKE_MESSAGE_CONTEXT "rapids.cpm.rapids_cpm_download_proprietary_binary")

# download and extract the binaries since they don't exist on the machine
include(FetchContent)
set(pkg_name "${package_name}_proprietary_binary")

if(POLICY CMP0135)
cmake_policy(SET CMP0135 NEW)
set(CMAKE_POLICY_DEFAULT_CMP0135 NEW)
endif()

FetchContent_Declare(${pkg_name} URL ${url})
FetchContent_MakeAvailable(${pkg_name})

# Tell the subsequent rapids_cpm_find where to search so that it uses this binary
set(${package_name}_ROOT "${${pkg_name}_SOURCE_DIR}" PARENT_SCOPE)
set(${package_name}_proprietary_binary ON PARENT_SCOPE)
endfunction()
42 changes: 17 additions & 25 deletions ...e/cpm/detail/get_proprietary_binary.cmake → ...m/detail/get_proprietary_binary_url.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#=============================================================================
# Copyright (c) 2022, NVIDIA CORPORATION.
# Copyright (c) 2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -16,22 +16,21 @@
include_guard(GLOBAL)

#[=======================================================================[.rst:
get_proprietary_binary
rapids_cpm_get_proprietary_binary_url
-------------------

.. versionadded:: v22.06.00
.. versionadded:: v23.04.00

Download the associated proprietary binary for the given project based on
the current CPU target architecture ( x86_64, aarch64, etc )
Generated the url for the associated proprietary binary for the given project based
robertmaynard marked this conversation as resolved.
Show resolved Hide resolved
on the current CPU target architecture ( x86_64, aarch64, etc )

.. note::
if override => the proprietary entry only in the override will be evaluated
if no override => the proprietary entry only in the default will be evaluated


#]=======================================================================]
function(rapids_cpm_get_proprietary_binary package_name version)
list(APPEND CMAKE_MESSAGE_CONTEXT "rapids.cpm.rapids_cpm_get_proprietary_binary")
function(rapids_cpm_get_proprietary_binary_url package_name version url_var)
list(APPEND CMAKE_MESSAGE_CONTEXT "rapids.cpm.rapids_cpm_get_proprietary_binary_url")

include("${rapids-cmake-dir}/cpm/detail/get_default_json.cmake")
include("${rapids-cmake-dir}/cpm/detail/get_override_json.cmake")
Expand Down Expand Up @@ -61,25 +60,18 @@ function(rapids_cpm_get_proprietary_binary package_name version)
include("${rapids-cmake-dir}/rapids-version.cmake")
endif()

# Determine the CUDA Toolkit version so that we properly evaluate the placeholders in
# `proprietary_binary`
if(proprietary_binary MATCHES "{cuda-toolkit-version")
find_package(CUDAToolkit REQUIRED)
set(cuda-toolkit-version ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR})
set(cuda-toolkit-version-major ${CUDAToolkit_VERSION_MAJOR})
endif()

# Evaluate any magic placeholders in the proprietary_binary value including the
# `rapids-cmake-version` value
cmake_language(EVAL CODE "set(proprietary_binary ${proprietary_binary})")

if(proprietary_binary)
# download and extract the binaries since they don't exist on the machine
include(FetchContent)
set(pkg_name "${package_name}_proprietary_binary")

if(POLICY CMP0135)
cmake_policy(SET CMP0135 NEW)
set(CMAKE_POLICY_DEFAULT_CMP0135 NEW)
endif()

FetchContent_Declare(${pkg_name} URL ${proprietary_binary})
FetchContent_MakeAvailable(${pkg_name})

# Tell the subsequent rapids_cpm_find where to search so that it uses this binary
set(${package_name}_ROOT "${${pkg_name}_SOURCE_DIR}" PARENT_SCOPE)
set(${package_name}_proprietary_binary ON PARENT_SCOPE)
endif()
# Tell the caller what the URL will be for this binary
set(${url_var} "${proprietary_binary}" PARENT_SCOPE)
endfunction()
8 changes: 6 additions & 2 deletions rapids-cmake/cpm/nvcomp.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,12 @@ function(rapids_cpm_nvcomp)
# first see if we have a proprietary pre-built binary listed in versions.json and it if requested.
set(nvcomp_proprietary_binary OFF) # will be set to true by rapids_cpm_get_proprietary_binary
if(_RAPIDS_USE_PROPRIETARY_BINARY)
include("${rapids-cmake-dir}/cpm/detail/get_proprietary_binary.cmake")
rapids_cpm_get_proprietary_binary(nvcomp ${version})
include("${rapids-cmake-dir}/cpm/detail/get_proprietary_binary_url.cmake")
include("${rapids-cmake-dir}/cpm/detail/download_proprietary_binary.cmake")
rapids_cpm_get_proprietary_binary_url(nvcomp ${version} nvcomp_url)
if(nvcomp_url)
rapids_cpm_download_proprietary_binary(nvcomp ${nvcomp_url})
endif()

# Record the nvcomp_DIR so that if USE_PROPRIETARY_BINARY is disabled we can safely clear the
# nvcomp_DIR value
Expand Down
4 changes: 4 additions & 0 deletions testing/cpm/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ add_cmake_config_test( cpm_nvcomp-simple.cmake )
add_cmake_config_test( cpm_nvcomp-invalid-arch.cmake )
add_cmake_config_test( cpm_nvcomp-override-clears-proprietary_binary.cmake )

add_cmake_config_test( cpm_proprietary-url-ctk-version-find-ctk.cmake )
add_cmake_config_test( cpm_proprietary-url-ctk-version.cmake )
add_cmake_config_test( cpm_proprietary-url-no-ctk-parsing.cmake )

add_cmake_config_test( cpm_rmm-export.cmake )
add_cmake_config_test( cpm_rmm-simple.cmake )

Expand Down
50 changes: 50 additions & 0 deletions testing/cpm/cpm_proprietary-url-ctk-version-find-ctk.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#=============================================================================
# Copyright (c) 2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#=============================================================================
include(${rapids-cmake-dir}/cpm/init.cmake)
include(${rapids-cmake-dir}/cpm/package_override.cmake)
include(${rapids-cmake-dir}/cpm/detail/get_proprietary_binary_url.cmake)
include(${rapids-cmake-dir}/cpm/detail/package_details.cmake)

rapids_cpm_init()

# Need to write out an override file with a proprietary blob url
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/override.json
[=[
{
"packages" : {
"test_binary" : {
"version" : "2.6.1",
"proprietary_binary" : {
"x86_64-linux" : "https://fake.url.com/${version}/${cuda-toolkit-version}/x86_64_${cuda-toolkit-version-major}.tgz",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume the answer is no since it's kind of the whole point of bracket expressions, but just in case: is there any way to force variable evaluation inside brackets? It would be nice to not need to repeat the URL every time.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean the fact that we have both x86 and arm? It is an explicit choice so that we can explicitly state support for multiple platforms in a easy to understand manner.

To have only a single url we would need:

  • Introduce a key named all_platforms
  • Add extra keys / values to proprietary_binary that mapped from the system arch ( x86 / arm ) to the placeholders used by that package in the download url

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, I just mean the repetition of the domain in the URL https://fake.url.com.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is a good idea. Want to create an issue for that and we can iterate on it independent of this PR?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#379

"aarch64-linux" : "https://fake.url.com/${version}/${cuda-toolkit-version}/aarch64_${cuda-toolkit-version-major}.tgz",
}
}
}
}
]=])
rapids_cpm_package_override(${CMAKE_CURRENT_BINARY_DIR}/override.json)

# Verify that the place-holders are evaluated correctly from `enable_language(CUDA)`
robertmaynard marked this conversation as resolved.
Show resolved Hide resolved
rapids_cpm_package_details(test_binary version repository tag shallow exclude)
rapids_cpm_get_proprietary_binary_url(test_binary ${version} url)

find_package(CUDAToolkit)
set(CTK_VER ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR})
set(CTK_VER_M ${CUDAToolkit_VERSION_MAJOR})
set(valid_url "https://fake.url.com/2.6.1/${CTK_VER}/${CMAKE_SYSTEM_PROCESSOR}_${CTK_VER_M}.tgz")
if(NOT valid_url STREQUAL url)
message(FATAL_ERROR "Expected: ${valid_url} got: ${url}")
endif()
51 changes: 51 additions & 0 deletions testing/cpm/cpm_proprietary-url-ctk-version.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#=============================================================================
# Copyright (c) 2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#=============================================================================
include(${rapids-cmake-dir}/cpm/init.cmake)
include(${rapids-cmake-dir}/cpm/package_override.cmake)
include(${rapids-cmake-dir}/cpm/detail/get_proprietary_binary_url.cmake)
include(${rapids-cmake-dir}/cpm/detail/package_details.cmake)

rapids_cpm_init()

# Need to write out an override file with a proprietary blob url
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/override.json
[=[
{
"packages" : {
"test_binary" : {
"version" : "2.6.1",
"proprietary_binary" : {
"x86_64-linux" : "https://fake.url.com/${version}/${cuda-toolkit-version}/x86_64_${cuda-toolkit-version-major}.tgz",
"aarch64-linux" : "https://fake.url.com/${version}/${cuda-toolkit-version}/aarch64_${cuda-toolkit-version-major}.tgz",
}
}
}
}
]=])
rapids_cpm_package_override(${CMAKE_CURRENT_BINARY_DIR}/override.json)

# Verify that the place-holders are evaluated correctly from `find_package(CUDAToolkit)`
robertmaynard marked this conversation as resolved.
Show resolved Hide resolved
find_package(CUDAToolkit REQUIRED)
rapids_cpm_package_details(test_binary version repository tag shallow exclude)
rapids_cpm_get_proprietary_binary_url(test_binary ${version} url)

find_package(CUDAToolkit)
robertmaynard marked this conversation as resolved.
Show resolved Hide resolved
set(CTK_VER ${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR})
set(CTK_VER_M ${CUDAToolkit_VERSION_MAJOR})
set(valid_url "https://fake.url.com/2.6.1/${CTK_VER}/${CMAKE_SYSTEM_PROCESSOR}_${CTK_VER_M}.tgz")
if(NOT valid_url STREQUAL url)
message(FATAL_ERROR "Expected: ${valid_url} got: ${url}")
endif()
46 changes: 46 additions & 0 deletions testing/cpm/cpm_proprietary-url-no-ctk-parsing.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#=============================================================================
# Copyright (c) 2023, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#=============================================================================
include(${rapids-cmake-dir}/cpm/init.cmake)
include(${rapids-cmake-dir}/cpm/package_override.cmake)
include(${rapids-cmake-dir}/cpm/detail/get_proprietary_binary_url.cmake)
include(${rapids-cmake-dir}/cpm/detail/package_details.cmake)

rapids_cpm_init()

# Need to write out an override file with a proprietary blob url
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/override.json
[=[
{
"packages" : {
"test_binary" : {
"version" : "2.6.1",
"proprietary_binary" : {
"x86_64-linux" : "https://fake.url.com/x86_${version}.tgz",
"aarch64-linux" : "https://fake.url.com/aarch_${version}.tgz",
}
}
}
}
]=])
rapids_cpm_package_override(${CMAKE_CURRENT_BINARY_DIR}/override.json)

rapids_cpm_package_details(test_binary version repository tag shallow exclude)
rapids_cpm_get_proprietary_binary_url(test_binary ${version} nvcomp_url)

# Verify that we didn't go searching for the CUDAToolkit
if(TARGET CUDA::cudart_static OR TARGET CUDA::cudart)
message(FATAL_ERROR "test_binary didn't use the cuda toolkit placeholder, but searching for it still occurred")
endif()
2 changes: 1 addition & 1 deletion testing/utils/project_template.cmake.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,5 +49,5 @@ add_subdirectory(${rapids-cmake_SOURCE_DIR} ${rapids-cmake_BINARY_DIR})

# We need to enable at least one language so that we get properly
# generated system search paths.
project(@test_name@ LANGUAGES CXX)
project(@test_name_stem@ LANGUAGES CXX)
robertmaynard marked this conversation as resolved.
Show resolved Hide resolved
include("@test_cmake_file@")