From 73797ab1d841223fc5912eee0f4a48f6e6770354 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Fri, 27 Sep 2024 14:17:40 -0400
Subject: [PATCH 01/12] Use nvcomp wheel instead of bundling nvcomp

Contributes to https://github.com/rapidsai/rapids-wheels-planning/issues/74
---
 dependencies.yaml             | 14 ++++++++++++++
 python/libcudf/pyproject.toml |  1 +
 2 files changed, 15 insertions(+)

diff --git a/dependencies.yaml b/dependencies.yaml
index ed36a23e5c3..001118a78f4 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -370,6 +370,20 @@ dependencies:
           # Align nvcomp version with rapids-cmake
           - nvcomp==4.0.1
           - spdlog>=1.14.1,<1.15
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+            packages:
+              - nvidia-nvcomp-cu12==4.0.1
+          - matrix:
+              cuda: "11.*"
+            packages:
+              - nvidia-nvcomp-cu11==4.0.1
+          - matrix:
+            packages:
+              - nvidia-nvcomp==4.0.1
   rapids_build_skbuild:
     common:
       - output_types: [conda, requirements, pyproject]
diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml
index 5bffe9fd96c..107d13567a9 100644
--- a/python/libcudf/pyproject.toml
+++ b/python/libcudf/pyproject.toml
@@ -69,4 +69,5 @@ requires = [
     "libkvikio==24.12.*,>=0.0.0a0",
     "librmm==24.12.*,>=0.0.0a0",
     "ninja",
+    "nvidia-nvcomp==4.0.1",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

From 9651827fba0235742692c25e6ff956aaf8f5f44d Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Fri, 27 Sep 2024 16:26:37 -0400
Subject: [PATCH 02/12] Don't install downloaded nvcomp, depend on wheel

---
 cpp/cmake/thirdparty/get_nvcomp.cmake | 4 +---
 dependencies.yaml                     | 8 ++++++++
 python/libcudf/pyproject.toml         | 4 +++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake
index 41bbf44abc8..f8488c515c8 100644
--- a/cpp/cmake/thirdparty/get_nvcomp.cmake
+++ b/cpp/cmake/thirdparty/get_nvcomp.cmake
@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -17,8 +17,6 @@ function(find_and_configure_nvcomp)
 
   include(${rapids-cmake-dir}/cpm/nvcomp.cmake)
   rapids_cpm_nvcomp(
-    BUILD_EXPORT_SET cudf-exports
-    INSTALL_EXPORT_SET cudf-exports
     USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP}
   )
 
diff --git a/dependencies.yaml b/dependencies.yaml
index 001118a78f4..f29847a2edd 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -152,6 +152,13 @@ files:
       - build_cpp
       - depends_on_libkvikio
       - depends_on_librmm
+  py_run_libcudf:
+    output: pyproject
+    pyproject_dir: python/libcudf
+    extras:
+      table: project
+    includes:
+      - depends_on_nvcomp
   py_build_pylibcudf:
     output: pyproject
     pyproject_dir: python/pylibcudf
@@ -370,6 +377,7 @@ dependencies:
           # Align nvcomp version with rapids-cmake
           - nvcomp==4.0.1
           - spdlog>=1.14.1,<1.15
+  depends_on_nvcomp:
     specific:
       - output_types: [requirements, pyproject]
         matrices:
diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml
index 107d13567a9..84660cbc276 100644
--- a/python/libcudf/pyproject.toml
+++ b/python/libcudf/pyproject.toml
@@ -37,6 +37,9 @@ classifiers = [
     "Programming Language :: C++",
     "Environment :: GPU :: NVIDIA CUDA",
 ]
+dependencies = [
+    "nvidia-nvcomp==4.0.1",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/cudf"
@@ -69,5 +72,4 @@ requires = [
     "libkvikio==24.12.*,>=0.0.0a0",
     "librmm==24.12.*,>=0.0.0a0",
     "ninja",
-    "nvidia-nvcomp==4.0.1",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

From dcf0b50bee0eca0b5f2364c35891c2cd7105e461 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Mon, 30 Sep 2024 11:16:57 -0400
Subject: [PATCH 03/12] Style

---
 cpp/cmake/thirdparty/get_nvcomp.cmake | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake
index f8488c515c8..1b6a1730161 100644
--- a/cpp/cmake/thirdparty/get_nvcomp.cmake
+++ b/cpp/cmake/thirdparty/get_nvcomp.cmake
@@ -16,9 +16,7 @@
 function(find_and_configure_nvcomp)
 
   include(${rapids-cmake-dir}/cpm/nvcomp.cmake)
-  rapids_cpm_nvcomp(
-    USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP}
-  )
+  rapids_cpm_nvcomp(USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP})
 
   # Per-thread default stream
   if(TARGET nvcomp AND CUDF_USE_PER_THREAD_DEFAULT_STREAM)

From 661d8343f7f2630799aa65c838d3c5e2b0e4bf3e Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Mon, 30 Sep 2024 12:17:05 -0400
Subject: [PATCH 04/12] Exclude nvcomp from auditwheel

---
 ci/build_wheel_libcudf.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh
index 8975381ceba..a52ae845344 100755
--- a/ci/build_wheel_libcudf.sh
+++ b/ci/build_wheel_libcudf.sh
@@ -10,6 +10,6 @@ package_dir="python/libcudf"
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 mkdir -p ${package_dir}/final_dist
-python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/*
+python -m auditwheel repair -w ${package_dir}/final_dist --exclude libnvcomp.so.4 ${package_dir}/dist/*
 
 RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist

From 723cdd06c1938ffd92383a51017b8094aa95a03e Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Mon, 30 Sep 2024 13:50:28 -0400
Subject: [PATCH 05/12] Set nvcomp rpath

---
 python/libcudf/CMakeLists.txt | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index 0a8f5c4807d..3ce4e63ce13 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -45,8 +45,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-# Ensure other libraries needed by libcudf.so get installed alongside it.
-include(cmake/Modules/WheelHelpers.cmake)
-install_aliased_imported_targets(
-  TARGETS cudf nvcomp::nvcomp DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
+set(rpaths
+  "$ORIGIN/../nvidia/nvcomp"
 )
+set_property(TARGET cudf PROPERTY INSTALL_RPATH ${rpaths} APPEND)

From 418344c76a546d2edd5e9c24f10539e758ed8b71 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Mon, 30 Sep 2024 13:53:48 -0400
Subject: [PATCH 06/12] Style

---
 python/libcudf/CMakeLists.txt | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index 3ce4e63ce13..3be64bc2afc 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -45,7 +45,9 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-set(rpaths
-  "$ORIGIN/../nvidia/nvcomp"
+set(rpaths "$ORIGIN/../nvidia/nvcomp")
+set_property(
+  TARGET cudf
+  PROPERTY INSTALL_RPATH ${rpaths}
+  APPEND
 )
-set_property(TARGET cudf PROPERTY INSTALL_RPATH ${rpaths} APPEND)

From b334dc57f4a099fde58b379f5fc16f59288457f1 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Mon, 30 Sep 2024 15:49:36 -0400
Subject: [PATCH 07/12] Fix rpath

---
 python/libcudf/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index 3be64bc2afc..e4447d5a1f6 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -45,7 +45,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-set(rpaths "$ORIGIN/../nvidia/nvcomp")
+set(rpaths "$ORIGIN/../../nvidia/nvcomp")
 set_property(
   TARGET cudf
   PROPERTY INSTALL_RPATH ${rpaths}

From 08f13691127997ca0ba758c4fe20e67ba0d479b7 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Mon, 30 Sep 2024 16:48:59 -0400
Subject: [PATCH 08/12] Make rpath usage configurable

---
 ci/build_wheel_libcudf.sh     |  1 +
 python/libcudf/CMakeLists.txt | 16 ++++++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh
index a52ae845344..ed446e9d187 100755
--- a/ci/build_wheel_libcudf.sh
+++ b/ci/build_wheel_libcudf.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 
 package_dir="python/libcudf"
 
+export SKBUILD_CMAKE_ARGS="-DUSE_CUDA_NVCOMP_WHEEL=ON"
 ./ci/build_wheel.sh ${package_dir}
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index e4447d5a1f6..bc314a0f4db 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -22,6 +22,8 @@ project(
   LANGUAGES CXX
 )
 
+option(USE_CUDA_NVCOMP_WHEEL "Use the CUDA nvcomp wheel instead of the system library" OFF)
+
 # Check if cudf is already available. If so, it is the user's responsibility to ensure that the
 # CMake package is also available at build time of the Python cudf package.
 find_package(cudf "${RAPIDS_VERSION}")
@@ -45,9 +47,11 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-set(rpaths "$ORIGIN/../../nvidia/nvcomp")
-set_property(
-  TARGET cudf
-  PROPERTY INSTALL_RPATH ${rpaths}
-  APPEND
-)
+if(USE_CUDA_NVCOMP_WHEEL)
+  set(rpaths "$ORIGIN/../../nvidia/nvcomp")
+  set_property(
+    TARGET cudf
+    PROPERTY INSTALL_RPATH ${rpaths}
+    APPEND
+  )
+endif()

From 1d875c3bd4e75f0c3ac475110953214f93e3ba7b Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Tue, 1 Oct 2024 09:46:10 -0400
Subject: [PATCH 09/12] Review feedback

---
 ci/build_wheel_libcudf.sh | 5 ++++-
 dependencies.yaml         | 8 ++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh
index ed446e9d187..f337410b146 100755
--- a/ci/build_wheel_libcudf.sh
+++ b/ci/build_wheel_libcudf.sh
@@ -11,6 +11,9 @@ export SKBUILD_CMAKE_ARGS="-DUSE_CUDA_NVCOMP_WHEEL=ON"
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 mkdir -p ${package_dir}/final_dist
-python -m auditwheel repair -w ${package_dir}/final_dist --exclude libnvcomp.so.4 ${package_dir}/dist/*
+python -m auditwheel repair \
+    --exclude libnvcomp.so.4 \
+    -w ${package_dir}/final_dist \
+    ${package_dir}/dist/*
 
 RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist
diff --git a/dependencies.yaml b/dependencies.yaml
index f29847a2edd..b192158c4ea 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -15,6 +15,7 @@ files:
       - depends_on_cupy
       - depends_on_libkvikio
       - depends_on_librmm
+      - depends_on_nvcomp
       - depends_on_rmm
       - develop
       - docs
@@ -374,10 +375,13 @@ dependencies:
           - fmt>=11.0.2,<12
           - flatbuffers==24.3.25
           - librdkafka>=2.5.0,<2.6.0a0
-          # Align nvcomp version with rapids-cmake
-          - nvcomp==4.0.1
           - spdlog>=1.14.1,<1.15
   depends_on_nvcomp:
+    common:
+      - output_types: conda
+        packages:
+          # Align nvcomp version with rapids-cmake
+          - nvcomp==4.0.1
     specific:
       - output_types: [requirements, pyproject]
         matrices:

From 69ee11e3f1a9a096a7046e48013deb7909610d26 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Tue, 1 Oct 2024 12:43:51 -0400
Subject: [PATCH 10/12] Update variable name

---
 ci/build_wheel_libcudf.sh     | 2 +-
 python/libcudf/CMakeLists.txt | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh
index f337410b146..d31de3c6eec 100755
--- a/ci/build_wheel_libcudf.sh
+++ b/ci/build_wheel_libcudf.sh
@@ -5,7 +5,7 @@ set -euo pipefail
 
 package_dir="python/libcudf"
 
-export SKBUILD_CMAKE_ARGS="-DUSE_CUDA_NVCOMP_WHEEL=ON"
+export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_WHEEL=ON"
 ./ci/build_wheel.sh ${package_dir}
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index bc314a0f4db..d0dfbbc82a6 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -22,7 +22,7 @@ project(
   LANGUAGES CXX
 )
 
-option(USE_CUDA_NVCOMP_WHEEL "Use the CUDA nvcomp wheel instead of the system library" OFF)
+option(USE_NVCOMP_WHEEL "Use the nvcomp wheel instead of the system library" OFF)
 
 # Check if cudf is already available. If so, it is the user's responsibility to ensure that the
 # CMake package is also available at build time of the Python cudf package.
@@ -47,7 +47,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-if(USE_CUDA_NVCOMP_WHEEL)
+if(USE_NVCOMP_WHEEL)
   set(rpaths "$ORIGIN/../../nvidia/nvcomp")
   set_property(
     TARGET cudf

From b72b0599909ec4df84d1e0de7499641c21e0c04e Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Tue, 1 Oct 2024 13:17:49 -0400
Subject: [PATCH 11/12] Update description

---
 python/libcudf/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index d0dfbbc82a6..0ce3239ba2e 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -22,7 +22,7 @@ project(
   LANGUAGES CXX
 )
 
-option(USE_NVCOMP_WHEEL "Use the nvcomp wheel instead of the system library" OFF)
+option(USE_NVCOMP_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF)
 
 # Check if cudf is already available. If so, it is the user's responsibility to ensure that the
 # CMake package is also available at build time of the Python cudf package.

From 319c95ee3d4dfcfe96f5fb97c58645539d8527e5 Mon Sep 17 00:00:00 2001
From: Kyle Edwards <kyedwards@nvidia.com>
Date: Tue, 1 Oct 2024 13:22:22 -0400
Subject: [PATCH 12/12] Update variable name

---
 ci/build_wheel_libcudf.sh     | 2 +-
 python/libcudf/CMakeLists.txt | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh
index d31de3c6eec..91bc071583e 100755
--- a/ci/build_wheel_libcudf.sh
+++ b/ci/build_wheel_libcudf.sh
@@ -5,7 +5,7 @@ set -euo pipefail
 
 package_dir="python/libcudf"
 
-export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_WHEEL=ON"
+export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON"
 ./ci/build_wheel.sh ${package_dir}
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt
index 0ce3239ba2e..2b208e2e021 100644
--- a/python/libcudf/CMakeLists.txt
+++ b/python/libcudf/CMakeLists.txt
@@ -22,7 +22,7 @@ project(
   LANGUAGES CXX
 )
 
-option(USE_NVCOMP_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF)
+option(USE_NVCOMP_RUNTIME_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF)
 
 # Check if cudf is already available. If so, it is the user's responsibility to ensure that the
 # CMake package is also available at build time of the Python cudf package.
@@ -47,7 +47,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-if(USE_NVCOMP_WHEEL)
+if(USE_NVCOMP_RUNTIME_WHEEL)
   set(rpaths "$ORIGIN/../../nvidia/nvcomp")
   set_property(
     TARGET cudf