Skip to content

Commit

Permalink
cuDF: Build CUDA 12 packages (#12922)
Browse files Browse the repository at this point in the history
This PR builds `libcudf` and `cudf` conda packages using CUDA 12. Resolves #12771.

Authors:
  - Bradley Dice (https://github.com/bdice)
  - AJ Schmidt (https://github.com/ajschmidt8)
  - Vyas Ramasubramani (https://github.com/vyasr)
  - https://github.com/jakirkham
  - Robert Maynard (https://github.com/robertmaynard)

Approvers:
  - https://github.com/jakirkham
  - Vyas Ramasubramani (https://github.com/vyasr)
  - Ray Douglass (https://github.com/raydouglass)

URL: #12922
  • Loading branch information
bdice authored Jun 23, 2023
1 parent 6aad528 commit 0688872
Show file tree
Hide file tree
Showing 19 changed files with 313 additions and 76 deletions.
16 changes: 8 additions & 8 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ concurrency:
jobs:
cpp-build:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand All @@ -37,7 +37,7 @@ jobs:
python-build:
needs: [cpp-build]
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand All @@ -46,7 +46,7 @@ jobs:
upload-conda:
needs: [cpp-build, python-build]
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand All @@ -57,7 +57,7 @@ jobs:
if: github.ref_type == 'branch'
needs: python-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
arch: "amd64"
branch: ${{ inputs.branch }}
Expand All @@ -69,7 +69,7 @@ jobs:
sha: ${{ inputs.sha }}
wheel-build-cudf:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand All @@ -81,7 +81,7 @@ jobs:
wheel-publish-cudf:
needs: wheel-build-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand All @@ -91,7 +91,7 @@ jobs:
wheel-build-dask-cudf:
needs: wheel-publish-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand All @@ -102,7 +102,7 @@ jobs:
wheel-publish-dask-cudf:
needs: wheel-build-dask-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@cuda-120
with:
build_type: ${{ inputs.build_type || 'branch' }}
branch: ${{ inputs.branch }}
Expand Down
28 changes: 14 additions & 14 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,49 +26,49 @@ jobs:
- wheel-build-dask-cudf
- wheel-tests-dask-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120
checks:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-120
with:
enable_check_generated_files: false
conda-cpp-build:
needs: checks
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120
with:
build_type: pull-request
conda-cpp-tests:
needs: conda-cpp-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120
with:
build_type: pull-request
conda-python-build:
needs: conda-cpp-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120
with:
build_type: pull-request
conda-python-cudf-tests:
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
with:
build_type: pull-request
test_script: "ci/test_python_cudf.sh"
conda-python-other-tests:
# Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
with:
build_type: pull-request
test_script: "ci/test_python_other.sh"
conda-java-tests:
needs: conda-cpp-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
build_type: pull-request
node_type: "gpu-v100-latest-1"
Expand All @@ -78,7 +78,7 @@ jobs:
conda-notebook-tests:
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
build_type: pull-request
node_type: "gpu-v100-latest-1"
Expand All @@ -88,7 +88,7 @@ jobs:
docs-build:
needs: conda-python-build
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
build_type: pull-request
node_type: "gpu-v100-latest-1"
Expand All @@ -98,7 +98,7 @@ jobs:
wheel-build-cudf:
needs: checks
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
with:
build_type: pull-request
package-name: cudf
Expand All @@ -107,7 +107,7 @@ jobs:
wheel-tests-cudf:
needs: wheel-build-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
with:
build_type: pull-request
package-name: cudf
Expand All @@ -116,7 +116,7 @@ jobs:
wheel-build-dask-cudf:
needs: wheel-tests-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@cuda-120
with:
build_type: pull-request
package-name: dask_cudf
Expand All @@ -125,7 +125,7 @@ jobs:
wheel-tests-dask-cudf:
needs: wheel-build-dask-cudf
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@cuda-120
with:
build_type: pull-request
package-name: dask_cudf
Expand Down
16 changes: 8 additions & 8 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ on:
jobs:
conda-cpp-tests:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
date: ${{ inputs.date }}
sha: ${{ inputs.sha }}
conda-cpp-memcheck-tests:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand All @@ -36,7 +36,7 @@ jobs:
run_script: "ci/test_cpp_memcheck.sh"
conda-python-cudf-tests:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand All @@ -46,7 +46,7 @@ jobs:
conda-python-other-tests:
# Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand All @@ -55,7 +55,7 @@ jobs:
test_script: "ci/test_python_other.sh"
conda-java-tests:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand All @@ -67,7 +67,7 @@ jobs:
run_script: "ci/test_java.sh"
conda-notebook-tests:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand All @@ -79,7 +79,7 @@ jobs:
run_script: "ci/test_notebooks.sh"
wheel-tests-cudf:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand All @@ -89,7 +89,7 @@ jobs:
test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests"
wheel-tests-dask-cudf:
secrets: inherit
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.08
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@cuda-120
with:
build_type: nightly
branch: ${{ inputs.branch }}
Expand Down
3 changes: 2 additions & 1 deletion ci/build_cpp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ rapids-print-env

rapids-logger "Begin cpp build"

rapids-mamba-retry mambabuild conda/recipes/libcudf
rapids-mamba-retry mambabuild \
conda/recipes/libcudf

rapids-upload-conda-to-s3 cpp
2 changes: 1 addition & 1 deletion ci/release/apply_wheel_modifications.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/das
sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml

if [[ $CUDA_SUFFIX == "-cu12" ]]; then
sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" python/cudf/pyproject.toml
sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" python/cudf/pyproject.toml
sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml
sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" python/{cudf,dask_cudf}/pyproject.toml
sed -i "/ptxcompiler/d" python/cudf/pyproject.toml
Expand Down
2 changes: 1 addition & 1 deletion conda/environments/all_cuda-118_arch-x86_64.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ dependencies:
- cachetools
- cmake>=3.26.4
- cubinlinker
- cuda-python>=11.7.1,<12.0
- cuda-python>=11.7.1,<12.0a0
- cuda-sanitizer-api=11.8.86
- cudatoolkit=11.8
- cupy>=12.0.0
Expand Down
95 changes: 95 additions & 0 deletions conda/environments/all_cuda-120_arch-x86_64.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
# This file is generated by `rapids-dependency-file-generator`.
# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
channels:
- rapidsai
- rapidsai-nightly
- dask/label/dev
- pytorch
- conda-forge
- nvidia
dependencies:
- aiobotocore>=2.2.0
- benchmark==1.8.0
- boto3>=1.21.21
- botocore>=1.24.21
- c-compiler
- cachetools
- cmake>=3.26.4
- cubinlinker
- cuda-cudart-dev
- cuda-nvcc
- cuda-nvrtc-dev
- cuda-python>=12.0,<13.0a0
- cuda-sanitizer-api
- cuda-version=12.0
- cupy>=12.0.0
- cxx-compiler
- cython>=0.29,<0.30
- dask-core>=2023.5.1
- dask-cuda==23.8.*
- dask>=2023.5.1
- distributed>=2023.5.1
- dlpack>=0.5,<0.6.0a0
- doxygen=1.8.20
- fastavro>=0.22.9
- fmt>=9.1.0,<10
- fsspec>=0.6.0
- gcc_linux-64=11.*
- gmock>=1.13.0
- gtest>=1.13.0
- hypothesis
- ipython
- libarrow==11.0.0.*
- libcufile-dev
- libcurand-dev
- libkvikio==23.8.*
- librdkafka>=1.9.0,<1.10.0a0
- librmm==23.8.*
- mimesis>=4.1.0
- moto>=4.0.8
- msgpack-python
- myst-nb
- nbsphinx
- ninja
- notebook
- numba>=0.57
- numpy>=1.21
- numpydoc
- nvtx>=0.2.1
- packaging
- pandas>=1.3,<1.6.0dev0
- pandoc
- pip
- pre-commit
- protobuf>=4.21.6,<4.22
- ptxcompiler
- pyarrow==11.0.0.*
- pydata-sphinx-theme
- pyorc
- pytest
- pytest-benchmark
- pytest-cases
- pytest-cov
- pytest-xdist
- python-confluent-kafka>=1.9.0,<1.10.0a0
- python-snappy>=0.6.0
- python>=3.9,<3.11
- pytorch<1.12.0
- rmm==23.8.*
- s3fs>=2022.3.0
- scikit-build>=0.13.1
- scipy
- spdlog>=1.11.0,<1.12
- sphinx
- sphinx-autobuild
- sphinx-copybutton
- sphinx-markdown-tables
- sphinxcontrib-websupport
- streamz
- sysroot_linux-64==2.17
- tokenizers==0.13.1
- transformers==4.24.0
- typing_extensions
- pip:
- git+https://github.com/python-streamz/streamz.git@master
name: all_cuda-120_arch-x86_64
3 changes: 3 additions & 0 deletions conda/recipes/cudf/conda_build_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,7 @@ cmake_version:
- ">=3.26.4"

cuda_compiler:
- cuda-nvcc

cuda11_compiler:
- nvcc
Loading

0 comments on commit 0688872

Please sign in to comment.