From 2f3e6295fdf537d5549ee319a6a5b507e08eec48 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 01:03:27 +0000 Subject: [PATCH 1/7] Don't set always_download by default in package_details --- rapids-cmake/cpm/detail/package_details.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/rapids-cmake/cpm/detail/package_details.cmake b/rapids-cmake/cpm/detail/package_details.cmake index cca327a4..1ece0d4d 100644 --- a/rapids-cmake/cpm/detail/package_details.cmake +++ b/rapids-cmake/cpm/detail/package_details.cmake @@ -92,7 +92,6 @@ function(rapids_cpm_package_details package_name version_var url_var tag_var sha set(exclude_from_all OFF) rapids_cpm_json_get_value(exclude_from_all) - set(always_download OFF) if(override_json_data AND json_data AND git_details_overridden) # `always_download` default value requires the package to exist in both the default and override # and that the git url / git tag have been modified. From 312d713d2227b8bd5a10af4392996ae7c95e2289 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 18:01:06 +0000 Subject: [PATCH 2/7] Unset always_download --- rapids-cmake/cpm/detail/package_details.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rapids-cmake/cpm/detail/package_details.cmake b/rapids-cmake/cpm/detail/package_details.cmake index 1ece0d4d..cc364ea1 100644 --- a/rapids-cmake/cpm/detail/package_details.cmake +++ b/rapids-cmake/cpm/detail/package_details.cmake @@ -92,6 +92,9 @@ function(rapids_cpm_package_details package_name version_var url_var tag_var sha set(exclude_from_all OFF) rapids_cpm_json_get_value(exclude_from_all) + # Ensure that always_download is not set by default so that the if(DEFINED always_download) check + # below works as expected in the default case. + unset(always_download) if(override_json_data AND json_data AND git_details_overridden) # `always_download` default value requires the package to exist in both the default and override # and that the git url / git tag have been modified. From 8a2004e976410fa548ee02a87cc558c191071570 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 18:19:37 +0000 Subject: [PATCH 3/7] Remove scikit-build from cache since we no longer use it --- testing/utils/fill_cache/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/utils/fill_cache/CMakeLists.txt b/testing/utils/fill_cache/CMakeLists.txt index 746d1e23..114a1efd 100644 --- a/testing/utils/fill_cache/CMakeLists.txt +++ b/testing/utils/fill_cache/CMakeLists.txt @@ -43,10 +43,6 @@ rapids_cpm_nvtx3(DOWNLOAD_ONLY ON) rapids_cpm_rmm(DOWNLOAD_ONLY ON) rapids_cpm_spdlog(DOWNLOAD_ONLY ON) rapids_cpm_fmt(DOWNLOAD_ONLY ON) -rapids_cpm_find(skbuild 0.14.1 - GIT_REPOSITORY https://github.com/scikit-build/scikit-build.git - GIT_TAG 0.14.1 - ) # Download all binary packages set(CPM_DOWNLOAD_ALL "OFF") From a3b21e4c58927755f7f7859e672805288097f63b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 18:47:38 +0000 Subject: [PATCH 4/7] Ensure that packages in the cache always have their source pulled from there. --- testing/utils/cmake_test.cmake | 7 +++++++ testing/utils/fill_cache/CMakeLists.txt | 11 +++++++++++ testing/utils/setup_cpm_cache.cmake | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/testing/utils/cmake_test.cmake b/testing/utils/cmake_test.cmake index e679ae43..4089ff47 100644 --- a/testing/utils/cmake_test.cmake +++ b/testing/utils/cmake_test.cmake @@ -96,6 +96,13 @@ function(add_cmake_test mode source_or_dir) if(DEFINED CPM_DOWNLOAD_LOCATION) list(APPEND extra_configure_flags "-DCPM_DOWNLOAD_LOCATION=${CPM_DOWNLOAD_LOCATION}") endif() + if(PACKAGES_IN_CPM_CACHE) + # Prevent ever finding preexisting built packages for those that we have in + # the cache. + foreach(pkg ${PACKAGES_IN_CPM_CACHE}) + list(APPEND extra_configure_flags "-DCPM_DOWNLOAD_${pkg}=ON") + endforeach() + endif() foreach(generator gen_name IN ZIP_LISTS supported_generators nice_gen_names) diff --git a/testing/utils/fill_cache/CMakeLists.txt b/testing/utils/fill_cache/CMakeLists.txt index 114a1efd..945a6da5 100644 --- a/testing/utils/fill_cache/CMakeLists.txt +++ b/testing/utils/fill_cache/CMakeLists.txt @@ -47,3 +47,14 @@ rapids_cpm_fmt(DOWNLOAD_ONLY ON) # Download all binary packages set(CPM_DOWNLOAD_ALL "OFF") rapids_cpm_nvcomp(USE_PROPRIETARY_BINARY ON DOWNLOAD_ONLY ON) + +# Print out all the packages in the cache +get_cmake_property(cpm_packages CACHE_VARIABLES) +set(packages) +foreach(p ${cpm_packages}) + if ("${p}" MATCHES "^CPM_PACKAGE_(.*)_SOURCE_DIR") + list(APPEND packages ${CMAKE_MATCH_1}) + endif() +endforeach() +string(REPLACE ";" ", " packages "${packages}") +message(STATUS "CPM packages in cache: {${packages}}") diff --git a/testing/utils/setup_cpm_cache.cmake b/testing/utils/setup_cpm_cache.cmake index 2272523d..c9a4b419 100644 --- a/testing/utils/setup_cpm_cache.cmake +++ b/testing/utils/setup_cpm_cache.cmake @@ -30,8 +30,17 @@ function(setup_cpm_cache ) -DCPM_SOURCE_CACHE=${CPM_SOURCE_CACHE} -DCPM_DOWNLOAD_LOCATION=${CPM_DOWNLOAD_LOCATION} WORKING_DIRECTORY ${src_dir} + OUTPUT_VARIABLE out_var ) + # Find the line in out_var that contains "CPM packages in cache + set(packages) + foreach(line IN LISTS out_var) + if("${line}" MATCHES "CPM packages in cache: {(.*)}") + string(REPLACE ", " ";" packages "${CMAKE_MATCH_1}") + endif() + endforeach() + set(PACKAGES_IN_CPM_CACHE ${packages} PARENT_SCOPE) set(CPM_SOURCE_CACHE "${CPM_SOURCE_CACHE}" PARENT_SCOPE) set(CPM_DOWNLOAD_LOCATION "${CPM_DOWNLOAD_LOCATION}" PARENT_SCOPE) endfunction() From 88185181b008c1ad6e1832afabe5dac94aa44ff7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 18:56:53 +0000 Subject: [PATCH 5/7] Use CPM_PACKAGE_* variable instead which always uses consistent casing with versions.json --- testing/cpm/verify_generated_pins.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/cpm/verify_generated_pins.cmake b/testing/cpm/verify_generated_pins.cmake index c75e788b..4627df4e 100644 --- a/testing/cpm/verify_generated_pins.cmake +++ b/testing/cpm/verify_generated_pins.cmake @@ -30,7 +30,7 @@ function(verify_generated_pins target_name) # only check projects that were downloaded by CPM (ignore those already in the build environment) foreach(proj IN LISTS _RAPIDS_PROJECTS) - if(${proj}_SOURCE_DIR) + if(CPM_PACKAGE_${proj}_SOURCE_DIR) list(APPEND projects-to-verify ${proj}) endif() endforeach() From f97b1dd7615507e2c3c4017b2b5faa9459ab3d2e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 19:21:48 +0000 Subject: [PATCH 6/7] Fix gtest test that explicitly requires avoiding downloading --- testing/cpm/cpm_find-gtest-no-gmock/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/cpm/cpm_find-gtest-no-gmock/CMakeLists.txt b/testing/cpm/cpm_find-gtest-no-gmock/CMakeLists.txt index 8102b1d9..6cdb49ed 100644 --- a/testing/cpm/cpm_find-gtest-no-gmock/CMakeLists.txt +++ b/testing/cpm/cpm_find-gtest-no-gmock/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,9 @@ include(${rapids-cmake-dir}/cpm/init.cmake) include(${rapids-cmake-dir}/cpm/gtest.cmake) set(CMAKE_PREFIX_PATH "${CMAKE_CURRENT_SOURCE_DIR}/mock_installed_gtest") +# Downloading is turned on by default for all packages in the CPM cache when +# testing, but in this case we specifically want it off. +set(CPM_DOWNLOAD_GTest OFF) rapids_cpm_init() rapids_cpm_gtest() From 52be0170d4d04d0a944cd69dc695b8a3516c393f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 4 Dec 2024 19:34:50 +0000 Subject: [PATCH 7/7] Now that we always download we should never be missing source directories --- testing/cpm/verify_generated_pins.cmake | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/testing/cpm/verify_generated_pins.cmake b/testing/cpm/verify_generated_pins.cmake index 4627df4e..014f1234 100644 --- a/testing/cpm/verify_generated_pins.cmake +++ b/testing/cpm/verify_generated_pins.cmake @@ -28,10 +28,9 @@ function(verify_generated_pins target_name) set(_RAPIDS_PIN_FILE "${CMAKE_CURRENT_BINARY_DIR}/rapids-cmake/pinned_versions.json") endif() - # only check projects that were downloaded by CPM (ignore those already in the build environment) foreach(proj IN LISTS _RAPIDS_PROJECTS) - if(CPM_PACKAGE_${proj}_SOURCE_DIR) - list(APPEND projects-to-verify ${proj}) + if(NOT CPM_PACKAGE_${proj}_SOURCE_DIR) + message(FATAL_ERROR "Attempting to verify a project that was not cloned as part of this build") endif() endforeach() @@ -39,7 +38,7 @@ function(verify_generated_pins target_name) COMMAND ${CMAKE_COMMAND} "-S${CMAKE_CURRENT_FUNCTION_LIST_DIR}/verify_generated_pins/" "-B${CMAKE_BINARY_DIR}/${target_name}_verify_build" "-Drapids-cmake-dir=${rapids-cmake-dir}" "-Dpinned_versions_file=${_RAPIDS_PIN_FILE}" - "-Dprojects-to-verify=${projects-to-verify}" + "-Dprojects-to-verify=${_RAPIDS_PROJECTS}" "-Dprojects-not-in-list=${_RAPIDS_PROJECTS_NOT_EXIST}" VERBATIM )