From d1e97b0c8a169896c63bb4833e1dccc1160cbcc0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 25 Jul 2024 00:12:10 -0500 Subject: [PATCH 1/2] split up CUDA-suffixed dependencies in dependencies.yaml (#5974) Contributes to https://github.com/rapidsai/build-planning/issues/31 In short, RAPIDS DLFW builds want to produce wheels with unsuffixed dependencies, e.g. `cudf` depending on `rmm`, not `rmm-cu12`. This PR is part of a series across all of RAPIDS to try to support that type of build by setting up CUDA-suffixed and CUDA-unsuffixed dependency lists in `dependencies.yaml`. For more details, see: * https://github.com/rapidsai/build-planning/issues/31#issuecomment-2245815818 * https://github.com/rapidsai/cudf/pull/16183 ## Notes for Reviewers ### Why target 24.08? This is targeting 24.08 because: 1. it should be very low-risk 2. getting these changes into 24.08 prevents the need to carry around patches for every library in DLFW builds using RAPIDS 24.08 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cuml/pull/5974 --- ci/release/update-version.sh | 13 +++--- dependencies.yaml | 76 +++++++++++++++++++++++++----------- python/cuml/pyproject.toml | 1 + 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 74054e7e15..52ed710f34 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -37,12 +37,6 @@ function sed_runner() { # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION -# pyproject.toml versions -sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml -sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml -sed_runner "s/pylibraft==.*\",/pylibraft==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml -sed_runner "s/raft-dask==.*\",/raft-dask==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\",/g" python/cuml/pyproject.toml - DEPENDENCIES=( cudf cuml @@ -59,10 +53,13 @@ DEPENDENCIES=( rapids-dask-dependency rmm ) -for FILE in dependencies.yaml conda/environments/*.yaml; do - for DEP in "${DEPENDENCIES[@]}"; do +for DEP in "${DEPENDENCIES[@]}"; do + for FILE in dependencies.yaml conda/environments/*.yaml; do sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}" done + for FILE in python/*/pyproject.toml; do + sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE} + done done sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" README.md diff --git a/dependencies.yaml b/dependencies.yaml index 53128a885a..4c4232cd5a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -180,8 +180,8 @@ dependencies: - &treelite treelite==4.3.0 - output_types: conda packages: - - &pylibraft_conda pylibraft==24.8.*,>=0.0.0a0 - - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &pylibraft_unsuffixed pylibraft==24.8.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -204,18 +204,22 @@ dependencies: - cuda-python - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - pylibraft-cu12==24.8.*,>=0.0.0a0 - rmm-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - - &pylibraft_cu11 pylibraft-cu11==24.8.*,>=0.0.0a0 - - &rmm_cu11 rmm-cu11==24.8.*,>=0.0.0a0 + - pylibraft-cu11==24.8.*,>=0.0.0a0 + - rmm-cu11==24.8.*,>=0.0.0a0 - matrix: packages: - - pylibraft==24.8.*,>=0.0.0a0 - - rmm==24.8.*,>=0.0.0a0 + - *pylibraft_unsuffixed + - *rmm_unsuffixed py_run: common: @@ -233,10 +237,10 @@ dependencies: - *treelite - output_types: conda packages: - - &cudf_conda cudf==24.8.*,>=0.0.0a0 - - &cupy_conda cupy>=12.0.0 - - &dask_cudf_conda dask-cudf==24.8.*,>=0.0.0a0 - - &raft_dask_conda raft-dask==24.8.*,>=0.0.0a0 + - &cudf_unsuffixed cudf==24.8.*,>=0.0.0a0 + - &cupy_unsuffixed cupy>=12.0.0 + - &dask_cudf_unsuffixed dask-cudf==24.8.*,>=0.0.0a0 + - &raft_dask_unsuffixed raft-dask==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -246,30 +250,58 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - cudf-cu12==24.8.*,>=0.0.0a0 - - cupy-cuda12x>=12.0.0 + - &cupy_pyproject_cu12 cupy-cuda12x>=12.0.0 - dask-cudf-cu12==24.8.*,>=0.0.0a0 - pylibraft-cu12==24.8.*,>=0.0.0a0 - raft-dask-cu12==24.8.*,>=0.0.0a0 - rmm-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: + - *cudf_unsuffixed + # NOTE: cupy still has a "-cuda12x" suffix here, because it's suffixed + # in DLFW builds + - *cupy_pyproject_cu12 + - *dask_cudf_unsuffixed + - *pylibraft_unsuffixed + - *raft_dask_unsuffixed + - *rmm_unsuffixed + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: &py_run_packages_cu11 - cudf-cu11==24.8.*,>=0.0.0a0 - &cupy_pyproject_cu11 cupy-cuda11x>=12.0.0 - dask-cudf-cu11==24.8.*,>=0.0.0a0 - - *pylibraft_cu11 + - pylibraft-cu11==24.8.*,>=0.0.0a0 - raft-dask-cu11==24.8.*,>=0.0.0a0 - - *rmm_cu11 + - rmm-cu11==24.8.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: + - *cudf_unsuffixed + # NOTE: cupy still has a "-cuda11x" suffix here, because it's suffixed + # in DLFW builds + - *cupy_pyproject_cu11 + - *dask_cudf_unsuffixed + - *pylibraft_unsuffixed + - *raft_dask_unsuffixed + - *rmm_unsuffixed - matrix: packages: - - *cudf_conda + - *cudf_unsuffixed - *cupy_pyproject_cu11 - - *dask_cudf_conda - - *pylibraft_conda - - *raft_dask_conda - - *rmm_conda + - *dask_cudf_unsuffixed + - *pylibraft_unsuffixed + - *raft_dask_unsuffixed + - *rmm_unsuffixed cuda_version: specific: - output_types: conda diff --git a/python/cuml/pyproject.toml b/python/cuml/pyproject.toml index 0d8554854b..90487b80b0 100644 --- a/python/cuml/pyproject.toml +++ b/python/cuml/pyproject.toml @@ -153,6 +153,7 @@ versioneer\.py | [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cuda-python", From 3e738537d5661f9b5a7322f3e74f43d6e30efbde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malte=20F=C3=B6rster?= <97973773+mfoerste4@users.noreply.github.com> Date: Fri, 26 Jul 2024 05:55:43 +0200 Subject: [PATCH 2/2] BUG: remove sample parameter from pca call to mean (#5980) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes a call to `raft::stats::mean()` deactivating the sample parameter in the pca code. CC @dantegd Authors: - Malte Förster (https://github.com/mfoerste4) Approvers: - Bradley Dice (https://github.com/bdice) - Dante Gama Dessavre (https://github.com/dantegd) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuml/pull/5980 --- cpp/src/pca/pca.cuh | 2 +- cpp/src/tsvd/tsvd.cuh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/pca/pca.cuh b/cpp/src/pca/pca.cuh index fcc83b22b5..5a59ff3db5 100644 --- a/cpp/src/pca/pca.cuh +++ b/cpp/src/pca/pca.cuh @@ -108,7 +108,7 @@ void pcaFit(const raft::handle_t& handle, auto n_components = prms.n_components; if (n_components > prms.n_cols) n_components = prms.n_cols; - raft::stats::mean(mu, input, prms.n_cols, prms.n_rows, true, false, stream); + raft::stats::mean(mu, input, prms.n_cols, prms.n_rows, false, false, stream); auto len = prms.n_cols * prms.n_cols; rmm::device_uvector cov(len, stream); diff --git a/cpp/src/tsvd/tsvd.cuh b/cpp/src/tsvd/tsvd.cuh index 543af045cb..b032b1a69a 100644 --- a/cpp/src/tsvd/tsvd.cuh +++ b/cpp/src/tsvd/tsvd.cuh @@ -271,21 +271,21 @@ void tsvdFitTransform(const raft::handle_t& handle, rmm::device_uvector mu_trans(prms.n_components, stream); raft::stats::mean( - mu_trans.data(), trans_input, prms.n_components, prms.n_rows, true, false, stream); + mu_trans.data(), trans_input, prms.n_components, prms.n_rows, false, false, stream); raft::stats::vars(explained_var, trans_input, mu_trans.data(), prms.n_components, prms.n_rows, - true, + false, false, stream); rmm::device_uvector mu(prms.n_cols, stream); rmm::device_uvector vars(prms.n_cols, stream); - raft::stats::mean(mu.data(), input, prms.n_cols, prms.n_rows, true, false, stream); - raft::stats::vars(vars.data(), input, mu.data(), prms.n_cols, prms.n_rows, true, false, stream); + raft::stats::mean(mu.data(), input, prms.n_cols, prms.n_rows, false, false, stream); + raft::stats::vars(vars.data(), input, mu.data(), prms.n_cols, prms.n_rows, false, false, stream); rmm::device_scalar total_vars(stream); raft::stats::sum(total_vars.data(), vars.data(), std::size_t(1), prms.n_cols, false, stream);