Use nvcomp wheel instead of bundling nvcomp

ci/build_wheel_libcudf.sh

@@ -5,11 +5,15 @@ set -euo pipefail
 
 package_dir="python/libcudf"
 
+export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON"
 ./ci/build_wheel.sh ${package_dir}
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 mkdir -p ${package_dir}/final_dist
-python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/*
+python -m auditwheel repair \
+ --exclude libnvcomp.so.4 \
+ -w ${package_dir}/final_dist \
+ ${package_dir}/dist/*
 
 RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist

cpp/cmake/thirdparty/get_nvcomp.cmake

@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.
+# Copyright (c) 2021-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -16,11 +16,7 @@
 function(find_and_configure_nvcomp)
 
  include(${rapids-cmake-dir}/cpm/nvcomp.cmake)
- rapids_cpm_nvcomp(
- BUILD_EXPORT_SET cudf-exports
- INSTALL_EXPORT_SET cudf-exports
- USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP}
- )
+ rapids_cpm_nvcomp(USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP})
 
  # Per-thread default stream
  if(TARGET nvcomp AND CUDF_USE_PER_THREAD_DEFAULT_STREAM)

dependencies.yaml

@@ -15,6 +15,7 @@ files:
  - depends_on_cupy
  - depends_on_libkvikio
  - depends_on_librmm
+ - depends_on_nvcomp
  - depends_on_rmm
  - develop
  - docs
@@ -152,6 +153,13 @@ files:
  - build_cpp
  - depends_on_libkvikio
  - depends_on_librmm
+ py_run_libcudf:
+ output: pyproject
+ pyproject_dir: python/libcudf
+ extras:
+ table: project
+ includes:
+ - depends_on_nvcomp
  py_build_pylibcudf:
  output: pyproject
  pyproject_dir: python/pylibcudf
@@ -367,9 +375,27 @@ dependencies:
  - fmt>=11.0.2,<12
  - flatbuffers==24.3.25
  - librdkafka>=2.5.0,<2.6.0a0
+ - spdlog>=1.14.1,<1.15
+ depends_on_nvcomp:
+ common:
+ - output_types: conda
+ packages:
  # Align nvcomp version with rapids-cmake
  - nvcomp==4.0.1
- - spdlog>=1.14.1,<1.15
+ specific:
+ - output_types: [requirements, pyproject]
+ matrices:
+ - matrix:
+ cuda: "12.*"
+ packages:
+ - nvidia-nvcomp-cu12==4.0.1
+ - matrix:
+ cuda: "11.*"
+ packages:
+ - nvidia-nvcomp-cu11==4.0.1
+ - matrix:
+ packages:
+ - nvidia-nvcomp==4.0.1
  rapids_build_skbuild:
  common:
  - output_types: [conda, requirements, pyproject]

python/libcudf/CMakeLists.txt

@@ -22,6 +22,8 @@ project(
  LANGUAGES CXX
 )
 
+option(USE_NVCOMP_RUNTIME_WHEEL "Use the nvcomp wheel at runtime instead of the system library" OFF)
+
 # Check if cudf is already available. If so, it is the user's responsibility to ensure that the
 # CMake package is also available at build time of the Python cudf package.
 find_package(cudf "${RAPIDS_VERSION}")
@@ -45,8 +47,11 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
 
-# Ensure other libraries needed by libcudf.so get installed alongside it.
-include(cmake/Modules/WheelHelpers.cmake)
-install_aliased_imported_targets(
- TARGETS cudf nvcomp::nvcomp DESTINATION ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
-)
+if(USE_NVCOMP_RUNTIME_WHEEL)
+ set(rpaths "$ORIGIN/../../nvidia/nvcomp")
+ set_property(
+ TARGET cudf
+ PROPERTY INSTALL_RPATH ${rpaths}
+ APPEND
+ )
+endif()

python/libcudf/pyproject.toml

@@ -37,6 +37,9 @@ classifiers = [
  "Programming Language :: C++",
  "Environment :: GPU :: NVIDIA CUDA",
 ]
+dependencies = [
+ "nvidia-nvcomp==4.0.1",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
 Homepage = "https://github.com/rapidsai/cudf"