From dd487ef1a6928efb75512329d79c986b25669674 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 09:47:10 -0700 Subject: [PATCH 01/72] Switch to using cuda 12 workflows --- .github/workflows/build.yaml | 16 ++++++++-------- .github/workflows/pr.yaml | 26 +++++++++++++------------- .github/workflows/test.yaml | 12 ++++++------ 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3c8cc4912d..f46432db10 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -57,7 +57,7 @@ jobs: if: github.ref_type == 'branch' && github.event_name == 'push' needs: python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 with: build_type: branch node_type: "gpu-latest-1" @@ -66,7 +66,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -79,7 +79,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-build-raft-dask: needs: wheel-publish-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -102,7 +102,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d0809e66b6..6a557ba52f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,41 +23,41 @@ jobs: - wheel-build-raft-dask - wheel-tests-raft-dask secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-120 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 with: build_type: pull-request docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 with: build_type: pull-request node_type: "gpu-latest-1" @@ -67,7 +67,7 @@ jobs: wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: pull-request package-name: pylibraft @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: pull-request package-name: pylibraft @@ -89,7 +89,7 @@ jobs: wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: pull-request package-name: raft_dask @@ -100,12 +100,12 @@ jobs: wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: pull-request package-name: raft_dask # Always want to test against latest dask/distributed. - test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@cuda-120 + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@cuda-120 test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e0731a9a97..cc4c7d849e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} @@ -44,13 +44,13 @@ jobs: test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test" wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: raft_dask - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - test-before-arm64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before-amd64: "pip install git+https://github.com/dask/dask.git@cuda-120 + test-before-arm64: "pip install git+https://github.com/dask/dask.git@cuda-120 test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" From 50d1576a10106c94e8414394d5a3c3fe7721afd7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 09:49:56 -0700 Subject: [PATCH 02/72] Add nvcc flags needed to work with broken nvidia packages --- conda/recipes/libraft/build_libraft.sh | 1 + conda/recipes/libraft/build_libraft_headers.sh | 1 + conda/recipes/libraft/build_libraft_nn_bench.sh | 1 + conda/recipes/libraft/build_libraft_template.sh | 3 ++- conda/recipes/libraft/build_libraft_tests.sh | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libraft/build_libraft.sh b/conda/recipes/libraft/build_libraft.sh index 2bf9b428cb..bd431f2c70 100644 --- a/conda/recipes/libraft/build_libraft.sh +++ b/conda/recipes/libraft/build_libraft.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. +export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libraft --allgpuarch --compile-lib --build-metrics --incl-cache-stats --no-nvtx diff --git a/conda/recipes/libraft/build_libraft_headers.sh b/conda/recipes/libraft/build_libraft_headers.sh index cc3b840e43..e730e39955 100644 --- a/conda/recipes/libraft/build_libraft_headers.sh +++ b/conda/recipes/libraft/build_libraft_headers.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. +export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libraft --allgpuarch --no-nvtx diff --git a/conda/recipes/libraft/build_libraft_nn_bench.sh b/conda/recipes/libraft/build_libraft_nn_bench.sh index dc6250f0f4..04b6b887af 100644 --- a/conda/recipes/libraft/build_libraft_nn_bench.sh +++ b/conda/recipes/libraft/build_libraft_nn_bench.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2023, NVIDIA CORPORATION. +export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh tests bench-ann --allgpuarch --no-nvtx cmake --install cpp/build --component ann_bench diff --git a/conda/recipes/libraft/build_libraft_template.sh b/conda/recipes/libraft/build_libraft_template.sh index 9759402884..8741b6e58a 100644 --- a/conda/recipes/libraft/build_libraft_template.sh +++ b/conda/recipes/libraft/build_libraft_template.sh @@ -2,4 +2,5 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. # Just building template so we verify it uses libraft.so and fail if it doesn't build -./build.sh template \ No newline at end of file +export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers +./build.sh template diff --git a/conda/recipes/libraft/build_libraft_tests.sh b/conda/recipes/libraft/build_libraft_tests.sh index cc28f93fb8..b35e99e4a1 100644 --- a/conda/recipes/libraft/build_libraft_tests.sh +++ b/conda/recipes/libraft/build_libraft_tests.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. +export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh tests bench-prims --allgpuarch --no-nvtx cmake --install cpp/build --component testing From 5502f7559d132c51c5ad66f095179586f494b30f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 10:18:05 -0700 Subject: [PATCH 03/72] Add a cuda11 compiler version and branch out the recipes --- conda/recipes/libraft/conda_build_config.yaml | 3 ++ conda/recipes/libraft/meta.yaml | 50 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 2a66f213a7..2fffca8c36 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -5,6 +5,9 @@ cxx_compiler_version: - 11 cuda_compiler: + - cuda-nvcc + +cuda11_compiler: - nvcc sysroot_version: diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index ccd7314484..352a5a1ae0 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -39,13 +39,21 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - librmm requirements: build: - {{ compiler('c') }} - {{ compiler('cxx') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -85,11 +93,19 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} requirements: build: - {{ compiler('c') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -119,11 +135,19 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} requirements: build: - {{ compiler('c') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -159,11 +183,19 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} requirements: build: - {{ compiler('c') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -195,11 +227,19 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} requirements: build: - {{ compiler('c') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja From d88346ec589301adfca477d8fa0c32001d6d450a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 16:32:56 -0700 Subject: [PATCH 04/72] Force build and test scripts to install rmm from the PR. --- ci/build_cpp.sh | 7 +++++-- ci/test_cpp.sh | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 853ae095d3..0d9f8db023 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. set -euo pipefail @@ -7,10 +7,13 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + rapids-print-env rapids-logger "Begin cpp build" -rapids-mamba-retry mambabuild conda/recipes/libraft +rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" conda/recipes/libraft rapids-upload-conda-to-s3 cpp diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index e32697a68a..f12f28446a 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,6 +18,9 @@ set +u conda activate test set -u +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -25,6 +28,7 @@ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-print-env rapids-mamba-retry install \ + --channel "${LIBRMM_CHANNEL}" \ --channel "${CPP_CHANNEL}" \ libraft-headers libraft libraft-tests From 6abba3790bc296fa0823ed363fd5f51b89b36920 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 16:42:49 -0700 Subject: [PATCH 05/72] Fix dask branch --- .github/workflows/pr.yaml | 6 ++---- .github/workflows/test.yaml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6a557ba52f..0ab40969f0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -100,12 +100,10 @@ jobs: wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - with: build_type: pull-request package-name: raft_dask # Always want to test against latest dask/distributed. - test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@cuda-120 - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@cuda-120 + test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cc4c7d849e..a213543469 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,12 +45,10 @@ jobs: wheel-tests-raft-dask: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - with: - build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: raft_dask - test-before-amd64: "pip install git+https://github.com/dask/dask.git@cuda-120 - test-before-arm64: "pip install git+https://github.com/dask/dask.git@cuda-120 + test-before-amd64: "pip install git+https://github.com/dask/dask.git@main + test-before-arm64: "pip install git+https://github.com/dask/dask.git@main test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" From 60058510b08bce8b9a43f50cdaf1901da8ab407f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 16:44:07 -0700 Subject: [PATCH 06/72] Put back build deleted lines to see if it fixes CI --- .github/workflows/pr.yaml | 2 ++ .github/workflows/test.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0ab40969f0..af7831d79f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -100,6 +100,8 @@ jobs: wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + with: build_type: pull-request package-name: raft_dask # Always want to test against latest dask/distributed. diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a213543469..6f3b78e63e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,6 +45,8 @@ jobs: wheel-tests-raft-dask: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + with: + build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} From 925dac1c23c769d2c89e562badd55f12b263922f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 16:46:44 -0700 Subject: [PATCH 07/72] Add missing quotes --- .github/workflows/pr.yaml | 4 ++-- .github/workflows/test.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index af7831d79f..a803d7b7c8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -105,7 +105,7 @@ jobs: build_type: pull-request package-name: raft_dask # Always want to test against latest dask/distributed. - test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main + test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6f3b78e63e..299f02b0a1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -51,6 +51,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: raft_dask - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main - test-before-arm64: "pip install git+https://github.com/dask/dask.git@main + test-before-amd64: "pip install git+https://github.com/dask/dask.git@main" + test-before-arm64: "pip install git+https://github.com/dask/dask.git@main" test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" From 823868ccf4cc261ad99047f2e80235913ad718dc Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 17:01:16 -0700 Subject: [PATCH 08/72] Temporarily comment out all unnecessary builds --- .github/workflows/pr.yaml | 148 +++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a803d7b7c8..eee526b9a8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,13 +15,13 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests - - conda-python-build - - conda-python-tests - - docs-build - - wheel-build-pylibraft - - wheel-tests-pylibraft - - wheel-build-raft-dask - - wheel-tests-raft-dask + #- conda-python-build + #- conda-python-tests + #- docs-build + #- wheel-build-pylibraft + #- wheel-tests-pylibraft + #- wheel-build-raft-dask + #- wheel-tests-raft-dask secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 checks: @@ -42,70 +42,70 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 - with: - build_type: pull-request - conda-python-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 - with: - build_type: pull-request - node_type: "gpu-latest-1" - arch: "amd64" - container_image: "rapidsai/ci:latest" - run_script: "ci/build_docs.sh" - wheel-build-pylibraft: - needs: checks - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - with: - build_type: pull-request - package-name: pylibraft - package-dir: python/pylibraft - skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" - uses-setup-env-vars: false - wheel-tests-pylibraft: - needs: wheel-build-pylibraft - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - with: - build_type: pull-request - package-name: pylibraft - test-before-amd64: "pip install cupy-cuda11x" - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test" - test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" - wheel-build-raft-dask: - needs: wheel-tests-pylibraft - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - with: - build_type: pull-request - package-name: raft_dask - package-dir: python/raft-dask - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" - skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" - uses-setup-env-vars: false - wheel-tests-raft-dask: - needs: wheel-build-raft-dask - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - with: - build_type: pull-request - package-name: raft_dask - # Always want to test against latest dask/distributed. - test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" - test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" - test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" + #conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 + # with: + # build_type: pull-request + #conda-python-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 + # with: + # build_type: pull-request + #docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + # with: + # build_type: pull-request + # node_type: "gpu-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci:latest" + # run_script: "ci/build_docs.sh" + #wheel-build-pylibraft: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: pylibraft + # package-dir: python/pylibraft + # skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" + # uses-setup-env-vars: false + #wheel-tests-pylibraft: + # needs: wheel-build-pylibraft + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: pylibraft + # test-before-amd64: "pip install cupy-cuda11x" + # # On arm also need to install cupy from the specific webpage. + # test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" + # test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test" + # test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" + #wheel-build-raft-dask: + # needs: wheel-tests-pylibraft + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: raft_dask + # package-dir: python/raft-dask + # before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + # skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" + # uses-setup-env-vars: false + #wheel-tests-raft-dask: + # needs: wheel-build-raft-dask + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: raft_dask + # # Always want to test against latest dask/distributed. + # test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" + # test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" + # test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" + # test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" From b88b5d0c674531c679fbdc2db30891bd36617a75 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 30 Mar 2023 17:08:48 -0700 Subject: [PATCH 09/72] Bifurcate all cuda deps by CUDA version --- conda/recipes/libraft/conda_build_config.yaml | 54 +++++-- conda/recipes/libraft/meta.yaml | 152 +++++++++++++----- 2 files changed, 157 insertions(+), 49 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 2fffca8c36..454155bfb0 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -34,40 +34,74 @@ h5py_version: nlohmann_json_version: - ">=3.11.2" +# CUDA 11 versions + # The CTK libraries below are missing from the conda-forge::cudatoolkit # package. The "*_host_*" version specifiers correspond to `11.8` packages and the # "*_run_*" version specifiers correspond to `11.x` packages. -libcublas_host_version: +cuda11_libcublas_host_version: - "=11.11.3.6" -libcublas_run_version: +cuda11_libcublas_run_version: - ">=11.5.2.43,<12.0.0" -libcurand_host_version: +cuda11_libcurand_host_version: - "=10.3.0.86" -libcurand_run_version: +cuda11_libcurand_run_version: - ">=10.2.5.43,<10.3.1" -libcusolver_host_version: +cuda11_libcusolver_host_version: - "=11.4.1.48" -libcusolver_run_version: +cuda11_libcusolver_run_version: - ">=11.2.0.43,<11.4.2" -libcusparse_host_version: +cuda11_libcusparse_host_version: - "=11.7.5.86" -libcusparse_run_version: +cuda11_libcusparse_run_version: - ">=11.6.0.43,<12.0.0" # `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all # architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the # "*_run_*" version specifiers correspond to `11.x` packages. -cuda_profiler_api_host_version: +cuda11_cuda_profiler_api_host_version: - "=11.8.86" -cuda_profiler_api_run_version: +cuda11_cuda_profiler_api_run_version: - ">=11.4.240,<12" + +# CUDA 12 versions + +libcublas_host_version: + - "=12.0.1.189,<13" + +libcublas_run_version: + - ">=12.0.1.189" + +libcurand_host_version: + - "=10.3.1.50" + +libcurand_run_version: + - ">=10.3.1.50" + +libcusolver_host_version: + - "=11.4.2.57" + +libcusolver_run_version: + - ">=11.4.2.57" + +libcusparse_host_version: + - "=12.0.0.76" + +libcusparse_run_version: + - ">=12.0.0.76" + +cuda_profiler_api_host_version: + - "=12.0.76" + +cuda_profiler_api_run_version: + - ">=12.0.76" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 352a5a1ae0..38e4ad5960 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -58,8 +58,19 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cuda-profiler-api {{ cuda_profiler_api_host_version }} + {% if cuda_major == "11" %} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - cudatoolkit ={{ cuda_version }} + - libcublas {{ cuda11_libcublas_host_version }} + - libcublas-dev {{ cuda11_libcublas_host_version }} + - libcurand {{ cuda11_libcurand_host_version }} + - libcurand-dev {{ cuda11_libcurand_host_version }} + - libcusolver {{ cuda11_libcusolver_host_version }} + - libcusolver-dev {{ cuda11_libcusolver_host_version }} + - libcusparse {{ cuda11_libcusparse_host_version }} + - libcusparse-dev {{ cuda11_libcusparse_host_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} - libcublas {{ libcublas_host_version }} - libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} @@ -68,9 +79,21 @@ outputs: - libcusolver-dev {{ libcusolver_host_version }} - libcusparse {{ libcusparse_host_version }} - libcusparse-dev {{ libcusparse_host_version }} + {% endif %} - librmm ={{ minor_version }} run: + {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} + - libcurand {{ cuda11_libcurand_run_version }} + - libcurand-dev {{ cuda11_libcurand_run_version }} + - libcusolver {{ cuda11_libcusolver_run_version }} + - libcusolver-dev {{ cuda11_libcusolver_run_version }} + - libcusparse {{ cuda11_libcusparse_run_version }} + - libcusparse-dev {{ cuda11_libcusparse_run_version }} + {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} @@ -80,6 +103,7 @@ outputs: - libcusolver-dev {{ libcusolver_run_version }} - libcusparse {{ libcusparse_run_version }} - libcusparse-dev {{ libcusparse_run_version }} + {% endif %} - librmm ={{ minor_version }} about: home: https://rapids.ai/ @@ -112,15 +136,28 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + {% if cuda_major == "11" %} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} + - libcurand {{ cuda11_libcurand_run_version }} + - libcurand-dev {{ cuda11_libcurand_run_version }} + - libcusolver {{ cuda11_libcusolver_run_version }} + - libcusolver-dev {{ cuda11_libcusolver_run_version }} + - libcusparse {{ cuda11_libcusparse_run_version }} + - libcusparse-dev {{ cuda11_libcusparse_run_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcurand {{ libcurand_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + {% endif %} run: - {{ pin_subpackage('libraft-headers', exact=True) }} about: @@ -155,17 +192,28 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + {% if cuda_major == "11" %} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} + - libcurand {{ cuda11_libcurand_run_version }} + - libcurand-dev {{ cuda11_libcurand_run_version }} + - libcusolver {{ cuda11_libcusolver_run_version }} + - libcusolver-dev {{ cuda11_libcusolver_run_version }} + - libcusparse {{ cuda11_libcusparse_run_version }} + - libcusparse-dev {{ cuda11_libcusparse_run_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcurand {{ libcurand_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + {% endif %} run: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} @@ -203,15 +251,28 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + {% if cuda_major == "11" %} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} + - libcurand {{ cuda11_libcurand_run_version }} + - libcurand-dev {{ cuda11_libcurand_run_version }} + - libcusolver {{ cuda11_libcusolver_run_version }} + - libcusolver-dev {{ cuda11_libcusolver_run_version }} + - libcusparse {{ cuda11_libcusparse_run_version }} + - libcusparse-dev {{ cuda11_libcusparse_run_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcurand {{ libcurand_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + {% endif %} run: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} @@ -247,15 +308,28 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + {% if cuda_major == "11" %} + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} + - libcurand {{ cuda11_libcurand_run_version }} + - libcurand-dev {{ cuda11_libcurand_run_version }} + - libcusolver {{ cuda11_libcusolver_run_version }} + - libcusolver-dev {{ cuda11_libcusolver_run_version }} + - libcusparse {{ cuda11_libcusparse_run_version }} + - libcusparse-dev {{ cuda11_libcusparse_run_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcurand {{ libcurand_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + {% endif %} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} - libfaiss>=1.7.1 From bc0bd3927db25244da5d0abb4f8f336577cf6aae Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 06:15:32 -0700 Subject: [PATCH 10/72] Add cudart --- conda/recipes/libraft/meta.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 38e4ad5960..be4b39eb7a 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -71,6 +71,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} - libcublas {{ libcublas_host_version }} - libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} @@ -95,6 +98,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -149,6 +155,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -205,6 +214,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -264,6 +276,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -321,6 +336,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} From a0ef11a5e6b692c0dad6c0a14402cfa9d3b19bd6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 06:44:20 -0700 Subject: [PATCH 11/72] Make sure to hard pin the cuda versions during build --- conda/recipes/libraft/meta.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index be4b39eb7a..3955cd48f9 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -71,9 +71,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} - libcublas {{ libcublas_host_version }} - libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} @@ -155,9 +155,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -214,9 +214,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -276,9 +276,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} @@ -336,9 +336,9 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} - libcublas {{ libcublas_run_version }} - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} From 60968ff5a0783df6a62c851ec7beea22f27125c7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 08:25:48 -0700 Subject: [PATCH 12/72] Update versions for 12.0.1 patch release of CUDA 12 --- conda/recipes/libraft/conda_build_config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 454155bfb0..d4e7381d41 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -77,28 +77,28 @@ cuda11_cuda_profiler_api_run_version: # CUDA 12 versions libcublas_host_version: - - "=12.0.1.189,<13" + - "=12.0.2.224" libcublas_run_version: - - ">=12.0.1.189" + - ">=12.0.2.224" libcurand_host_version: - - "=10.3.1.50" + - "=10.3.1.124" libcurand_run_version: - - ">=10.3.1.50" + - ">=10.3.1.124" libcusolver_host_version: - - "=11.4.2.57" + - "=11.4.3.51" libcusolver_run_version: - - ">=11.4.2.57" + - ">=11.4.3.51" libcusparse_host_version: - - "=12.0.0.76" + - "=12.0.1.140" libcusparse_run_version: - - ">=12.0.0.76" + - ">=12.0.1.140" cuda_profiler_api_host_version: - "=12.0.76" From f32bf157b828a4aa205e6ab6a5128738732a9b3e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 12:56:45 -0700 Subject: [PATCH 13/72] Pin librmm --- conda/recipes/libraft/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 3955cd48f9..1123b251d5 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -83,7 +83,7 @@ outputs: - libcusparse {{ libcusparse_host_version }} - libcusparse-dev {{ libcusparse_host_version }} {% endif %} - - librmm ={{ minor_version }} + - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* run: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} @@ -110,7 +110,7 @@ outputs: - libcusparse {{ libcusparse_run_version }} - libcusparse-dev {{ libcusparse_run_version }} {% endif %} - - librmm ={{ minor_version }} + - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* about: home: https://rapids.ai/ license: Apache-2.0 From e7d204fe4c99b800cc7f65505852875f04ca754a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 12:57:35 -0700 Subject: [PATCH 14/72] Fix libcusolver version --- conda/recipes/libraft/conda_build_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index d4e7381d41..54e352c61e 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -89,10 +89,10 @@ libcurand_run_version: - ">=10.3.1.124" libcusolver_host_version: - - "=11.4.3.51" + - "=11.4.3.1" libcusolver_run_version: - - ">=11.4.3.51" + - ">=11.4.3.1" libcusparse_host_version: - "=12.0.1.140" From 6e5e321eeafdb9c551a0d5719ec41efc1761b781 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 13:00:29 -0700 Subject: [PATCH 15/72] Revert to 12.0.0 versions --- conda/recipes/libraft/conda_build_config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 54e352c61e..b4713cf569 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -77,28 +77,28 @@ cuda11_cuda_profiler_api_run_version: # CUDA 12 versions libcublas_host_version: - - "=12.0.2.224" + - "=12.0.1.189" libcublas_run_version: - - ">=12.0.2.224" + - ">=12.0.1.189" libcurand_host_version: - - "=10.3.1.124" + - "=10.3.1.50" libcurand_run_version: - - ">=10.3.1.124" + - ">=10.3.1.50" libcusolver_host_version: - - "=11.4.3.1" + - "=11.4.2.57" libcusolver_run_version: - - ">=11.4.3.1" + - ">=11.4.2.57" libcusparse_host_version: - - "=12.0.1.140" + - "=12.0.0.76" libcusparse_run_version: - - ">=12.0.1.140" + - ">=12.0.0.76" cuda_profiler_api_host_version: - "=12.0.76" From 221b877dee2f8ed8c69e5e227b64f750ed253039 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 13:51:10 -0700 Subject: [PATCH 16/72] Add cuda runtime to build env for nvcc to see --- conda/recipes/libraft/meta.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 1123b251d5..042a2ddd5f 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -53,6 +53,9 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} {% endif %} - cmake {{ cmake_version }} - ninja @@ -135,6 +138,9 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} @@ -193,6 +199,9 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} @@ -255,6 +264,9 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} From 593619efb5fbd9ebd4c15f408c54665352bb5f25 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 31 Mar 2023 15:40:14 -0700 Subject: [PATCH 17/72] Add missing deps --- conda/recipes/libraft/meta.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 042a2ddd5f..6b49100deb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -191,6 +191,9 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} {% endif %} requirements: build: @@ -327,6 +330,9 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} From ac329ff30a603bc914a7bead1e3c43e05dc098c0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 3 Apr 2023 06:43:00 -0700 Subject: [PATCH 18/72] Force using nvidia cublas --- conda/recipes/libraft/meta.yaml | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 6b49100deb..52dedd7909 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -64,8 +64,8 @@ outputs: {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - cudatoolkit ={{ cuda_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} + - nvidia::libcublas {{ cuda11_libcublas_host_version }} + - nvidia::libcublas-dev {{ cuda11_libcublas_host_version }} - libcurand {{ cuda11_libcurand_host_version }} - libcurand-dev {{ cuda11_libcurand_host_version }} - libcusolver {{ cuda11_libcusolver_host_version }} @@ -77,8 +77,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} + - nvidia::libcublas {{ libcublas_host_version }} + - nvidia::libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} - libcurand-dev {{ libcurand_host_version }} - libcusolver {{ libcusolver_host_version }} @@ -91,8 +91,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_run_version }} - - libcublas-dev {{ cuda11_libcublas_run_version }} + - nvidia::libcublas {{ cuda11_libcublas_run_version }} + - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -104,8 +104,8 @@ outputs: - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} - cuda-cudart-static {{ cuda_spec }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} + - nvidia::libcublas {{ libcublas_run_version }} + - nvidia::libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -151,8 +151,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_run_version }} - - libcublas-dev {{ cuda11_libcublas_run_version }} + - nvidia::libcublas {{ cuda11_libcublas_run_version }} + - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -164,8 +164,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} + - nvidia::libcublas {{ libcublas_run_version }} + - nvidia::libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -216,8 +216,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_run_version }} - - libcublas-dev {{ cuda11_libcublas_run_version }} + - nvidia::libcublas {{ cuda11_libcublas_run_version }} + - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -229,8 +229,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} + - nvidia::libcublas {{ libcublas_run_version }} + - nvidia::libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -281,8 +281,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_run_version }} - - libcublas-dev {{ cuda11_libcublas_run_version }} + - nvidia::libcublas {{ cuda11_libcublas_run_version }} + - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -294,8 +294,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} + - nvidia::libcublas {{ libcublas_run_version }} + - nvidia::libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -344,8 +344,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_run_version }} - - libcublas-dev {{ cuda11_libcublas_run_version }} + - nvidia::libcublas {{ cuda11_libcublas_run_version }} + - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -357,8 +357,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} + - nvidia::libcublas {{ libcublas_run_version }} + - nvidia::libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} From 91855528f1fb34886656f6c93e692b8b0eb14bc1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 3 Apr 2023 07:14:02 -0700 Subject: [PATCH 19/72] Revert "Force using nvidia cublas" This reverts commit ac329ff30a603bc914a7bead1e3c43e05dc098c0. --- conda/recipes/libraft/meta.yaml | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 52dedd7909..6b49100deb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -64,8 +64,8 @@ outputs: {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - cudatoolkit ={{ cuda_version }} - - nvidia::libcublas {{ cuda11_libcublas_host_version }} - - nvidia::libcublas-dev {{ cuda11_libcublas_host_version }} + - libcublas {{ cuda11_libcublas_host_version }} + - libcublas-dev {{ cuda11_libcublas_host_version }} - libcurand {{ cuda11_libcurand_host_version }} - libcurand-dev {{ cuda11_libcurand_host_version }} - libcusolver {{ cuda11_libcusolver_host_version }} @@ -77,8 +77,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - nvidia::libcublas {{ libcublas_host_version }} - - nvidia::libcublas-dev {{ libcublas_host_version }} + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} - libcurand {{ libcurand_host_version }} - libcurand-dev {{ libcurand_host_version }} - libcusolver {{ libcusolver_host_version }} @@ -91,8 +91,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - nvidia::libcublas {{ cuda11_libcublas_run_version }} - - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -104,8 +104,8 @@ outputs: - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} - cuda-cudart-static {{ cuda_spec }} - - nvidia::libcublas {{ libcublas_run_version }} - - nvidia::libcublas-dev {{ libcublas_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -151,8 +151,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - nvidia::libcublas {{ cuda11_libcublas_run_version }} - - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -164,8 +164,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - nvidia::libcublas {{ libcublas_run_version }} - - nvidia::libcublas-dev {{ libcublas_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -216,8 +216,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - nvidia::libcublas {{ cuda11_libcublas_run_version }} - - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -229,8 +229,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - nvidia::libcublas {{ libcublas_run_version }} - - nvidia::libcublas-dev {{ libcublas_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -281,8 +281,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - nvidia::libcublas {{ cuda11_libcublas_run_version }} - - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -294,8 +294,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - nvidia::libcublas {{ libcublas_run_version }} - - nvidia::libcublas-dev {{ libcublas_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} @@ -344,8 +344,8 @@ outputs: {% if cuda_major == "11" %} - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - nvidia::libcublas {{ cuda11_libcublas_run_version }} - - nvidia::libcublas-dev {{ cuda11_libcublas_run_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} @@ -357,8 +357,8 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - nvidia::libcublas {{ libcublas_run_version }} - - nvidia::libcublas-dev {{ libcublas_run_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} - libcurand {{ libcurand_run_version }} - libcurand-dev {{ libcurand_run_version }} - libcusolver {{ libcusolver_run_version }} From 6952fc9db7639bd501c64681c9e076daae345746 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 3 Apr 2023 07:18:49 -0700 Subject: [PATCH 20/72] Try pinning cuda-version --- conda/recipes/libraft/meta.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 6b49100deb..7f317454c4 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -73,6 +73,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_host_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -100,6 +101,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} @@ -160,6 +162,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -225,6 +228,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -290,6 +294,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -353,6 +358,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} From 303f3dced48e9a8e902286297f137ec46d3d79fc Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 3 Apr 2023 07:37:40 -0700 Subject: [PATCH 21/72] Fix spec --- conda/recipes/libraft/meta.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 7f317454c4..d05fe1f2bb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -73,7 +73,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-version != 12.0 # Temporary hack + - cuda-version !=12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_host_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -101,7 +101,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack + - cuda-version !=12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} @@ -162,7 +162,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack + - cuda-version !=12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -228,7 +228,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack + - cuda-version !=12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -294,7 +294,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack + - cuda-version !=12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -358,7 +358,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack + - cuda-version !=12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} From 3c7fa81e5c43cc862d435e5d9df9e75827afb411 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 4 Apr 2023 11:00:31 -0700 Subject: [PATCH 22/72] Revert "Fix spec" This reverts commit 303f3dced48e9a8e902286297f137ec46d3d79fc. --- conda/recipes/libraft/meta.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index d05fe1f2bb..7f317454c4 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -73,7 +73,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-version !=12.0 # Temporary hack + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_host_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -101,7 +101,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version !=12.0 # Temporary hack + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} @@ -162,7 +162,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version !=12.0 # Temporary hack + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -228,7 +228,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version !=12.0 # Temporary hack + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -294,7 +294,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version !=12.0 # Temporary hack + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -358,7 +358,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version !=12.0 # Temporary hack + - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} From bebd60c7bf15c9a53d2597a539d88eccb1ccfb6a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 4 Apr 2023 11:00:34 -0700 Subject: [PATCH 23/72] Revert "Try pinning cuda-version" This reverts commit 6952fc9db7639bd501c64681c9e076daae345746. --- conda/recipes/libraft/meta.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 7f317454c4..6b49100deb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -73,7 +73,6 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_host_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -101,7 +100,6 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} @@ -162,7 +160,6 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -228,7 +225,6 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -294,7 +290,6 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} @@ -358,7 +353,6 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-version != 12.0 # Temporary hack - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} From f3761b8c0489f692d705c9f40001e55f9c535e51 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 4 Apr 2023 11:12:20 -0700 Subject: [PATCH 24/72] Try reordering channels --- ci/build_cpp.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 0d9f8db023..7fedc28e82 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -10,6 +10,25 @@ export CMAKE_GENERATOR=Ninja RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then + cat << EOF > /opt/conda/.condarc + auto_update_conda: False + channels: + - rapidsai + - rapidsai-nightly + - dask/label/dev + - pytorch + - nvidia + - conda-forge + conda-build: + set_build_id: false + root_dir: /tmp/conda-bld-workspace + output_folder: /tmp/conda-bld-output + number_channel_notices: 0 + always_yes: true + EOF +fi + rapids-print-env rapids-logger "Begin cpp build" From 85d45fd7de2efd7875e92831b4b24b5acf5a751b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 4 Apr 2023 11:24:30 -0700 Subject: [PATCH 25/72] Fix indent --- ci/build_cpp.sh | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 7fedc28e82..decd3698a0 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -11,22 +11,22 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then - cat << EOF > /opt/conda/.condarc - auto_update_conda: False - channels: - - rapidsai - - rapidsai-nightly - - dask/label/dev - - pytorch - - nvidia - - conda-forge - conda-build: - set_build_id: false - root_dir: /tmp/conda-bld-workspace - output_folder: /tmp/conda-bld-output - number_channel_notices: 0 - always_yes: true - EOF +cat << EOF > /opt/conda/.condarc +auto_update_conda: False +channels: +- rapidsai +- rapidsai-nightly +- dask/label/dev +- pytorch +- nvidia +- conda-forge +conda-build: +set_build_id: false +root_dir: /tmp/conda-bld-workspace +output_folder: /tmp/conda-bld-output +number_channel_notices: 0 +always_yes: true +EOF fi rapids-print-env From 0b26f47ce9c7a5bf79be98a949711102113153cc Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 4 Apr 2023 11:42:50 -0700 Subject: [PATCH 26/72] Remove any hidden characters --- ci/build_cpp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index decd3698a0..4fcdc61a19 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,9 +20,9 @@ channels: - pytorch - nvidia - conda-forge -conda-build: +conda_build: set_build_id: false -root_dir: /tmp/conda-bld-workspace +root_dir: /tmp/conda-bld/workspace output_folder: /tmp/conda-bld-output number_channel_notices: 0 always_yes: true From 830fe9e5e2e4c159a0272a9c1a5ff0e6bdfae8bf Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 4 Apr 2023 11:53:15 -0700 Subject: [PATCH 27/72] Use the correct root dir --- ci/build_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 4fcdc61a19..80f1b3283e 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -22,7 +22,7 @@ channels: - conda-forge conda_build: set_build_id: false -root_dir: /tmp/conda-bld/workspace +root_dir: /opt/conda output_folder: /tmp/conda-bld-output number_channel_notices: 0 always_yes: true From a66d561272f4bad0003488e0fe8bff1bd7f85bab Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 5 Apr 2023 17:11:31 -0700 Subject: [PATCH 28/72] Remove faiss dependence --- conda/recipes/libraft/meta.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 6b49100deb..a9cf6b4c33 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -368,14 +368,14 @@ outputs: {% endif %} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} - - libfaiss>=1.7.1 - - faiss-proc=*=cuda + #- libfaiss>=1.7.1 + #- faiss-proc=*=cuda run: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - glog {{ glog_version }} - - faiss-proc=*=cuda - - libfaiss {{ faiss_version }} + #- faiss-proc=*=cuda + #- libfaiss {{ faiss_version }} - h5py {{ h5py_version }} about: home: https://rapids.ai/ From 1f7a69e255366694245e52f698f50c64a99e2d42 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 5 Apr 2023 17:56:02 -0700 Subject: [PATCH 29/72] Remove faiss from bmarks --- cpp/bench/ann/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 6267be518e..27d9db2c18 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -15,9 +15,9 @@ # ################################################################################################## # * compiler function ----------------------------------------------------------------------------- -option(RAFT_ANN_BENCH_USE_FAISS_BFKNN "Include faiss' brute-force knn algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" ON) -option(RAFT_ANN_BENCH_USE_FAISS_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" ON) +option(RAFT_ANN_BENCH_USE_FAISS_BFKNN "Include faiss' brute-force knn algorithm in benchmark" OFF) +option(RAFT_ANN_BENCH_USE_FAISS_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" OFF) +option(RAFT_ANN_BENCH_USE_FAISS_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" OFF) option(RAFT_ANN_BENCH_USE_RAFT_BFKNN "Include raft's brute-force knn algorithm in benchmark" ON) option(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT "Include raft's ivf flat algorithm in benchmark" ON) option(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ "Include raft's ivf pq algorithm in benchmark" ON) From f1f61a81be7f8d7bf83446c1a3f807b74d1aba0f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 5 Apr 2023 21:56:29 -0700 Subject: [PATCH 30/72] Fix patched condarc --- ci/build_cpp.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 80f1b3283e..7dfd68a533 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -14,18 +14,18 @@ if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then cat << EOF > /opt/conda/.condarc auto_update_conda: False channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- pytorch -- nvidia -- conda-forge -conda_build: -set_build_id: false -root_dir: /opt/conda -output_folder: /tmp/conda-bld-output -number_channel_notices: 0 + - rapidsai + - rapidsai-nightly + - dask/label/dev + - pytorch + - nvidia + - conda-forge always_yes: true +number_channel_notices: 0 +conda_build: + set_build_id: false + root_dir: /tmp/conda-bld-workspace + output_folder: /tmp/conda-bld-output EOF fi From 5e0000375a481396a6680aa711bc9ba0802c7cbe Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 07:35:02 -0700 Subject: [PATCH 31/72] Disable faiss in a more elegant manner that still runs the tests on CUDA 11 --- conda/recipes/libraft/meta.yaml | 14 ++++++++++---- cpp/bench/ann/CMakeLists.txt | 13 ++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index a9cf6b4c33..2b5fea4641 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -368,14 +368,20 @@ outputs: {% endif %} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} - #- libfaiss>=1.7.1 - #- faiss-proc=*=cuda + # Temporarily ignore faiss benchmarks on CUDA 12 + {% if cuda_major == "11" %} + - libfaiss>=1.7.1 + - faiss-proc=*=cuda + {% endif %} run: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} - glog {{ glog_version }} - #- faiss-proc=*=cuda - #- libfaiss {{ faiss_version }} + # Temporarily ignore faiss benchmarks on CUDA 12 + {% if cuda_major == "11" %} + - faiss-proc=*=cuda + - libfaiss {{ faiss_version }} + {% endif %} - h5py {{ h5py_version }} about: home: https://rapids.ai/ diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 27d9db2c18..a2c71111e3 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -15,9 +15,9 @@ # ################################################################################################## # * compiler function ----------------------------------------------------------------------------- -option(RAFT_ANN_BENCH_USE_FAISS_BFKNN "Include faiss' brute-force knn algorithm in benchmark" OFF) -option(RAFT_ANN_BENCH_USE_FAISS_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" OFF) -option(RAFT_ANN_BENCH_USE_FAISS_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" OFF) +option(RAFT_ANN_BENCH_USE_FAISS_BFKNN "Include faiss' brute-force knn algorithm in benchmark" ON) +option(RAFT_ANN_BENCH_USE_FAISS_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" ON) +option(RAFT_ANN_BENCH_USE_FAISS_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" ON) option(RAFT_ANN_BENCH_USE_RAFT_BFKNN "Include raft's brute-force knn algorithm in benchmark" ON) option(RAFT_ANN_BENCH_USE_RAFT_IVF_FLAT "Include raft's ivf flat algorithm in benchmark" ON) option(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ "Include raft's ivf pq algorithm in benchmark" ON) @@ -26,6 +26,13 @@ option(RAFT_ANN_BENCH_USE_GGNN "Include ggnn algorithm in benchmark" ON) find_package(Threads REQUIRED) +# Disable faiss benchmarks on CUDA 12 since faiss is not yet CUDA 12-enabled. +if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0.0) + set(RAFT_ANN_BENCH_USE_FAISS_BFKNN OFF) + set(RAFT_ANN_BENCH_USE_FAISS_IVF_FLAT OFF) + set(RAFT_ANN_BENCH_USE_FAISS_IVF_PQ OFF) +endif() + set(RAFT_ANN_BENCH_USE_FAISS OFF) if(RAFT_ANN_BENCH_USE_FAISS_BFKNN OR RAFT_ANN_BENCH_USE_FAISS_IVFPQ From 6445d045071ba12671115c15fdc2885c578064ff Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 07:42:23 -0700 Subject: [PATCH 32/72] Add CUDA 12 to dependencies.yaml for testing --- dependencies.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 0460e2dd81..9f80687028 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -162,6 +162,21 @@ dependencies: specific: - output_types: conda matrices: + - matrix: + cuda: "12.0" + packages: + - cuda-cudart=12.0 + - cuda-cudart-dev=12.0 + - cuda-cudart-static=12.0 + - cuda-profiler-api=12.0.76 + - libcublas-dev=12.0.1.189 + - libcublas=12.0.1.189 + - libcurand-dev=10.3.1.50 + - libcurand=10.3.1.50 + - libcusolver-dev=11.4.2.57 + - libcusolver=11.4.2.57 + - libcusparse-dev=12.0.0.76 + - libcusparse=12.0.0.76 - matrix: cuda: "11.8" packages: From 7bddaee79fe9deecad99f51095b43c044d73dbc2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 10:10:36 -0700 Subject: [PATCH 33/72] Add nvjitlink to run requirements for CUDA 12 --- conda/recipes/libraft/conda_build_config.yaml | 6 ++++++ conda/recipes/libraft/meta.yaml | 2 ++ 2 files changed, 8 insertions(+) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index b4713cf569..5573f3dd98 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -105,3 +105,9 @@ cuda_profiler_api_host_version: cuda_profiler_api_run_version: - ">=12.0.76" + +libnvjitlink_host_version: + - "=12.0.140" + +libnvjitlink_run_version: + - ">=12.0.140" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 2b5fea4641..3e26216094 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -85,6 +85,7 @@ outputs: - libcusolver-dev {{ libcusolver_host_version }} - libcusparse {{ libcusparse_host_version }} - libcusparse-dev {{ libcusparse_host_version }} + - libnvjitlink {{ libnvjitlink_host_version }} {% endif %} - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* run: @@ -112,6 +113,7 @@ outputs: - libcusolver-dev {{ libcusolver_run_version }} - libcusparse {{ libcusparse_run_version }} - libcusparse-dev {{ libcusparse_run_version }} + - libnvjitlink {{ libnvjitlink_run_version }} {% endif %} - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* about: From 4031a668f63125440a72bbab641ff5aecdee68bb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 13 Jun 2023 12:03:09 -0700 Subject: [PATCH 34/72] Remove changes to pr.yaml --- .github/workflows/pr.yaml | 148 +++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index eee526b9a8..a803d7b7c8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,13 +15,13 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests - #- conda-python-build - #- conda-python-tests - #- docs-build - #- wheel-build-pylibraft - #- wheel-tests-pylibraft - #- wheel-build-raft-dask - #- wheel-tests-raft-dask + - conda-python-build + - conda-python-tests + - docs-build + - wheel-build-pylibraft + - wheel-tests-pylibraft + - wheel-build-raft-dask + - wheel-tests-raft-dask secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 checks: @@ -42,70 +42,70 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: pull-request - #conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 - # with: - # build_type: pull-request - #conda-python-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 - # with: - # build_type: pull-request - #docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 - # with: - # build_type: pull-request - # node_type: "gpu-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci:latest" - # run_script: "ci/build_docs.sh" - #wheel-build-pylibraft: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: pylibraft - # package-dir: python/pylibraft - # skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" - # uses-setup-env-vars: false - #wheel-tests-pylibraft: - # needs: wheel-build-pylibraft - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: pylibraft - # test-before-amd64: "pip install cupy-cuda11x" - # # On arm also need to install cupy from the specific webpage. - # test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - # test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test" - # test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" - #wheel-build-raft-dask: - # needs: wheel-tests-pylibraft - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: raft_dask - # package-dir: python/raft-dask - # before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" - # skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" - # uses-setup-env-vars: false - #wheel-tests-raft-dask: - # needs: wheel-build-raft-dask - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: raft_dask - # # Always want to test against latest dask/distributed. - # test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" - # test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" - # test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" - # test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" + wheel-build-pylibraft: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + with: + build_type: pull-request + package-name: pylibraft + package-dir: python/pylibraft + skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" + uses-setup-env-vars: false + wheel-tests-pylibraft: + needs: wheel-build-pylibraft + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + with: + build_type: pull-request + package-name: pylibraft + test-before-amd64: "pip install cupy-cuda11x" + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" + test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test" + test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" + wheel-build-raft-dask: + needs: wheel-tests-pylibraft + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + with: + build_type: pull-request + package-name: raft_dask + package-dir: python/raft-dask + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" + uses-setup-env-vars: false + wheel-tests-raft-dask: + needs: wheel-build-raft-dask + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + with: + build_type: pull-request + package-name: raft_dask + # Always want to test against latest dask/distributed. + test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft-dep && pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl && pip install git+https://github.com/dask/dask.git@main" + test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test" + test-smoketest: "python ./ci/wheel_smoke_test_raft_dask.py" From cf1a793d6d580b0acf1eb210fe3dbbbde3f6e05f Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 26 Jun 2023 18:16:34 -0500 Subject: [PATCH 35/72] ENH Remove NVCC temp flags and librmm CI channel --- ci/build_cpp.sh | 24 +------------------ ci/test_cpp.sh | 4 ---- conda/recipes/libraft/build_libraft.sh | 1 - .../recipes/libraft/build_libraft_headers.sh | 1 - .../recipes/libraft/build_libraft_nn_bench.sh | 1 - .../recipes/libraft/build_libraft_template.sh | 1 - conda/recipes/libraft/build_libraft_tests.sh | 1 - 7 files changed, 1 insertion(+), 32 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 7dfd68a533..e4b1d7fafb 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,32 +7,10 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - -if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then -cat << EOF > /opt/conda/.condarc -auto_update_conda: False -channels: - - rapidsai - - rapidsai-nightly - - dask/label/dev - - pytorch - - nvidia - - conda-forge -always_yes: true -number_channel_notices: 0 -conda_build: - set_build_id: false - root_dir: /tmp/conda-bld-workspace - output_folder: /tmp/conda-bld-output -EOF -fi - rapids-print-env rapids-logger "Begin cpp build" -rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" conda/recipes/libraft +rapids-mamba-retry mambabuild conda/recipes/libraft rapids-upload-conda-to-s3 cpp diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index f12f28446a..e32697a68a 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,9 +18,6 @@ set +u conda activate test set -u -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -28,7 +25,6 @@ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-print-env rapids-mamba-retry install \ - --channel "${LIBRMM_CHANNEL}" \ --channel "${CPP_CHANNEL}" \ libraft-headers libraft libraft-tests diff --git a/conda/recipes/libraft/build_libraft.sh b/conda/recipes/libraft/build_libraft.sh index aee4acd71c..7d4173e8bb 100644 --- a/conda/recipes/libraft/build_libraft.sh +++ b/conda/recipes/libraft/build_libraft.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. -export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libraft --allgpuarch --compile-lib --build-metrics=compile_lib --incl-cache-stats --no-nvtx diff --git a/conda/recipes/libraft/build_libraft_headers.sh b/conda/recipes/libraft/build_libraft_headers.sh index e730e39955..cc3b840e43 100644 --- a/conda/recipes/libraft/build_libraft_headers.sh +++ b/conda/recipes/libraft/build_libraft_headers.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. -export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libraft --allgpuarch --no-nvtx diff --git a/conda/recipes/libraft/build_libraft_nn_bench.sh b/conda/recipes/libraft/build_libraft_nn_bench.sh index 91c88f4111..00078792a1 100644 --- a/conda/recipes/libraft/build_libraft_nn_bench.sh +++ b/conda/recipes/libraft/build_libraft_nn_bench.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash # Copyright (c) 2023, NVIDIA CORPORATION. -export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh bench-ann --allgpuarch --no-nvtx --build-metrics=bench_ann --incl-cache-stats cmake --install cpp/build --component ann_bench diff --git a/conda/recipes/libraft/build_libraft_template.sh b/conda/recipes/libraft/build_libraft_template.sh index 8741b6e58a..bd7719af76 100644 --- a/conda/recipes/libraft/build_libraft_template.sh +++ b/conda/recipes/libraft/build_libraft_template.sh @@ -2,5 +2,4 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. # Just building template so we verify it uses libraft.so and fail if it doesn't build -export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh template diff --git a/conda/recipes/libraft/build_libraft_tests.sh b/conda/recipes/libraft/build_libraft_tests.sh index 43f2e50c41..05a2b59eb0 100644 --- a/conda/recipes/libraft/build_libraft_tests.sh +++ b/conda/recipes/libraft/build_libraft_tests.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. -export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh tests bench-prims --allgpuarch --no-nvtx --build-metrics=tests_bench_prims --incl-cache-stats cmake --install cpp/build --component testing From 34dbf370acd3f5ff2861696ffcbb9fc40b9083c5 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 26 Jun 2023 20:09:49 -0500 Subject: [PATCH 36/72] ENH Update recipes meta --- .../all_cuda-118_arch-x86_64.yaml | 3 +- .../all_cuda-120_arch-x86_64.yaml | 61 +++++++++++++++++++ .../bench_ann_cuda-118_arch-x86_64.yaml | 3 +- conda/recipes/libraft/meta.yaml | 13 ++-- conda/recipes/pylibraft/meta.yaml | 18 +++++- conda/recipes/raft-dask/meta.yaml | 17 +++++- dependencies.yaml | 38 ++++++------ 7 files changed, 123 insertions(+), 30 deletions(-) create mode 100644 conda/environments/all_cuda-120_arch-x86_64.yaml diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8fe8f16522..fe06e21b16 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -14,7 +14,8 @@ dependencies: - cmake>=3.23.1,!=3.25.0 - cuda-profiler-api=11.8.86 - cuda-python>=11.7.1,<12.0 -- cudatoolkit=11.8 +- cuda-version=11.8 +- cudatoolkit - cupy>=12.0.0 - cxx-compiler - cython>=0.29,<0.30 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml new file mode 100644 index 0000000000..320fec177b --- /dev/null +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -0,0 +1,61 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- dask/label/dev +- conda-forge +- nvidia +dependencies: +- breathe +- c-compiler +- clang-tools=16.0.1 +- clang=16.0.1 +- cmake>=3.23.1,!=3.25.0 +- cuda-cudart +- cuda-cudart-dev +- cuda-profiler-api +- cuda-python>=11.7.1,<12.0 +- cuda-version=12.0 +- cupy>=12.0.0 +- cxx-compiler +- cython>=0.29,<0.30 +- dask-core>=2023.5.1 +- dask-cuda==23.8.* +- dask>=2023.5.1 +- distributed>=2023.5.1 +- doxygen>=1.8.20 +- gcc_linux-64=11.* +- gmock>=1.13.0 +- graphviz +- gtest>=1.13.0 +- ipython +- joblib>=0.11 +- libcublas +- libcublas-dev +- libcurand +- libcurand-dev +- libcusolver +- libcusolver-dev +- libcusparse +- libcusparse-dev +- nccl>=2.9.9 +- ninja +- numba>=0.57 +- numpy>=1.21 +- numpydoc +- pydata-sphinx-theme +- pytest +- pytest-cov +- recommonmark +- rmm==23.8.* +- scikit-build>=0.13.1 +- scikit-learn +- scipy +- sphinx-copybutton +- sphinx-markdown-tables +- sysroot_linux-64==2.17 +- ucx-proc=*=gpu +- ucx-py==0.33.* +- ucx>=1.13.0 +name: all_cuda-120_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml index 2013c16fa4..74b966cc03 100644 --- a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml @@ -12,7 +12,8 @@ dependencies: - clang=16.0.1 - cmake>=3.23.1,!=3.25.0 - cuda-profiler-api=11.8.86 -- cudatoolkit=11.8 +- cuda-version=11.8 +- cudatoolkit - cxx-compiler - cython>=0.29,<0.30 - faiss-proc=*=cuda diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 985f071344..828047df3c 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -56,31 +56,30 @@ outputs: - {{ compiler('cuda') }} {{ cuda_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} {% endif %} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: {% if cuda_major == "11" %} - - cudatoolkit ={{ cuda_version }} + - cudatoolkit {% else %} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - libnvjitlink {{ libnvjitlink_host_version }} {% endif %} - - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* + - cuda-version ={{ cuda_version }} + - librmm ={{ minor_version }} run: {% if cuda_major == "11" %} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cudatoolkit {% else %} - cuda-cudart {{ cuda_spec }} - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} - libnvjitlink {{ libnvjitlink_run_version }} {% endif %} - - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - librmm ={{ minor_version }} about: home: https://rapids.ai/ license: Apache-2.0 diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 7730801801..1bf49540fa 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -26,13 +26,22 @@ requirements: build: - {{ compiler('c') }} - {{ compiler('cxx') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: + {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0 - - cudatoolkit ={{ cuda_version }} + - cudatoolkit + {% else} + - cuda-python >=12.0,<13.0a0 + {% endif} + - cuda-version ={{ cuda_version } - cython >=0.29,<0.30 - libraft {{ version }} - libraft-headers {{ version }} @@ -42,8 +51,13 @@ requirements: - scikit-build >=0.13.1 - setuptools run: - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0 + - cudatoolkit + {% else} + - cuda-python >=12.0,<13.0a0 + {% endif} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - libraft {{ version }} - libraft-headers {{ version }} - python x.x diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 26c0eed4f9..981c079290 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -26,13 +26,21 @@ requirements: build: - {{ compiler('c') }} - {{ compiler('cxx') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: + {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0 - - cudatoolkit ={{ cuda_version }} + - cudatoolkit + {% else} + - cuda-python >=12.0,<13.0a0 + {% endif} - cython >=0.29,<0.30 - nccl >=2.9.9 - pylibraft {{ version }} @@ -44,8 +52,13 @@ requirements: - ucx-proc=*=gpu - ucx-py {{ ucx_py_version }} run: - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0 + - cudatoolkit + {% else} + - cuda-python >=12.0,<13.0a0 + {% endif} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask >=2023.5.1 - dask-core >=2023.5.1 - dask-cuda ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 7dec90499f..43cd437f2e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.8"] + cuda: ["11.8", "12.0"] arch: [x86_64] includes: - build @@ -168,22 +168,23 @@ dependencies: - matrix: cuda: "12.0" packages: - - cuda-cudart=12.0 - - cuda-cudart-dev=12.0 - - cuda-cudart-static=12.0 - - cuda-profiler-api=12.0.76 - - libcublas-dev=12.0.1.189 - - libcublas=12.0.1.189 - - libcurand-dev=10.3.1.50 - - libcurand=10.3.1.50 - - libcusolver-dev=11.4.2.57 - - libcusolver=11.4.2.57 - - libcusparse-dev=12.0.0.76 - - libcusparse=12.0.0.76 + - cuda-version=12.0 + - cuda-cudart + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcublas + - libcurand-dev + - libcurand + - libcusolver-dev + - libcusolver + - libcusparse-dev + - libcusparse - matrix: cuda: "11.8" packages: - - cudatoolkit=11.8 + - cuda-version=11.8 + - cudatoolkit - cuda-profiler-api=11.8.86 - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 @@ -196,7 +197,8 @@ dependencies: - matrix: cuda: "11.5" packages: - - cudatoolkit=11.5 + - cuda-version=11.5 + - cudatoolkit - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - libcublas-dev>=11.7.3.1,<=11.7.4.6 - libcublas>=11.7.3.1,<=11.7.4.6 @@ -209,7 +211,8 @@ dependencies: - matrix: cuda: "11.4" packages: - - cudatoolkit=11.4 + - cuda-version=11.4 + - cudatoolkit - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages - &libcublas_dev114 libcublas-dev>=11.5.2.43,<=11.6.5.2 - &libcublas114 libcublas>=11.5.2.43,<=11.6.5.2 @@ -222,7 +225,8 @@ dependencies: - matrix: cuda: "11.2" packages: - - cudatoolkit=11.2 + - cuda-version=11.2 + - cudatoolkit - cuda-profiler-api>=11.4.240,<=11.8.86 # use any `11.x` version since pkg is missing several CUDA/arch packages # The NVIDIA channel doesn't publish pkgs older than 11.4 for these libs, # so 11.2 uses 11.4 packages (the oldest available). From c15458abf3f745e2daf9c4cb4251c40336e55d6b Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 26 Jun 2023 20:21:02 -0500 Subject: [PATCH 37/72] FIX forgot to commit libraft meta changes --- conda/recipes/libraft/meta.yaml | 130 ++++++++++++++++---------------- 1 file changed, 64 insertions(+), 66 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 828047df3c..d70e2267d5 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -96,7 +96,7 @@ outputs: run: - {{ pin_subpackage('libraft-headers-only', exact=True) }} {% if cuda_major == "11" %} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_run_version }} - libcublas-dev {{ cuda11_libcublas_run_version }} @@ -107,17 +107,18 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_run_version }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} - - libcurand {{ libcurand_run_version }} - - libcurand-dev {{ libcurand_run_version }} - - libcusolver {{ libcusolver_run_version }} - - libcusolver-dev {{ libcusolver_run_version }} - - libcusparse {{ libcusparse_run_version }} - - libcusparse-dev {{ libcusparse_run_version }} - - libnvjitlink {{ libnvjitlink_run_version }} + - cuda-profiler-api + - libcublas + - libcublas-dev + - libcurand + - libcurand-dev + - libcusolver + - libcusolver-dev + - libcusparse + - libcusparse-dev + - libnvjitlink {% endif %} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} about: home: https://rapids.ai/ license: Apache-2.0 @@ -144,7 +145,6 @@ outputs: - {{ compiler('cuda') }} {{ cuda_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} @@ -153,7 +153,7 @@ outputs: host: - {{ pin_subpackage('libraft-headers', exact=True) }} {% if cuda_major == "11" %} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -164,19 +164,19 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - cuda-profiler-api + - cuda-cudart + - cuda-cudart-dev + - libcublas + - libcublas-dev + - libcurand + - libcurand-dev + - libcusolver + - libcusolver-dev + - libcusparse + - libcusparse-dev {% endif %} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} run: - {{ pin_subpackage('libraft-headers', exact=True) }} about: @@ -197,7 +197,6 @@ outputs: - {{ compiler('cuda') }} {{ cuda_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} {% endif %} requirements: build: @@ -208,7 +207,6 @@ outputs: - {{ compiler('cuda') }} {{ cuda_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} @@ -217,7 +215,7 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} {% if cuda_major == "11" %} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -228,19 +226,20 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - cuda-profiler-api + - cuda-cudart + - cuda-cudart-dev + - cuda-cudart-static + - libcublas + - libcublas-dev + - libcurand + - libcurand-dev + - libcusolver + - libcusolver-dev + - libcusparse + - libcusparse-dev {% endif %} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} run: @@ -273,7 +272,6 @@ outputs: - {{ compiler('cuda') }} {{ cuda_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} @@ -294,19 +292,19 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - cuda-profiler-api + - cuda-cudart + - cuda-cudart-dev + - libcublas + - libcublas-dev + - libcurand + - libcurand-dev + - libcusolver + - libcusolver-dev + - libcusparse + - libcusparse-dev {% endif %} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} run: - {{ pin_subpackage('libraft', exact=True) }} about: @@ -335,7 +333,6 @@ outputs: - {{ compiler('cuda') }} {{ cuda_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} @@ -355,19 +352,20 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - cuda-profiler-api + - cuda-cudart + - cuda-cudart-dev + - cuda-cudart-static + - libcublas + - libcublas-dev + - libcurand + - libcurand-dev + - libcusolver + - libcusolver-dev + - libcusparse + - libcusparse-dev {% endif %} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} # Temporarily ignore faiss benchmarks on CUDA 12 From 9db2c1ec9b1df7a7c6e444b0f86b321cc87cf3af Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 26 Jun 2023 20:54:42 -0500 Subject: [PATCH 38/72] FIX Missing librmm line in libraft-headers meta section --- conda/recipes/libraft/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index d70e2267d5..aabca32d69 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -95,6 +95,8 @@ outputs: requirements: run: - {{ pin_subpackage('libraft-headers-only', exact=True) }} + - librmm ={{ minor_version }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} @@ -118,7 +120,6 @@ outputs: - libcusparse-dev - libnvjitlink {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} about: home: https://rapids.ai/ license: Apache-2.0 From 256d2c8b2422000e2fd1b5db06e5265c08e87fb1 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 26 Jun 2023 21:34:47 -0500 Subject: [PATCH 39/72] FIX Small fixes in libraft-headers section --- conda/recipes/libraft/meta.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index aabca32d69..e35d33d82e 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -90,13 +90,17 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + ignore_run_exports_from: + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} {{ cuda_version }} + {% endif %} - librmm requirements: run: - {{ pin_subpackage('libraft-headers-only', exact=True) }} - librmm ={{ minor_version }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} From 7a277acb34307af295de15b06206b84597f41bac Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 26 Jun 2023 22:02:08 -0500 Subject: [PATCH 40/72] FIX Mode small fixes in libraft-headers section --- conda/recipes/libraft/meta.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index e35d33d82e..e4c0fbcf37 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -74,9 +74,9 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - libnvjitlink {{ libnvjitlink_run_version }} + - cuda-cudart + - cuda-cudart-dev + - libnvjitlink {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - librmm ={{ minor_version }} @@ -113,6 +113,8 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} + - cuda-cudart + - cuda-cudart-dev - cuda-profiler-api - libcublas - libcublas-dev From 3377f1abb1422e707b7a74e1e88537fd79943e22 Mon Sep 17 00:00:00 2001 From: Tamas Bela Feher Date: Tue, 27 Jun 2023 16:44:29 +0200 Subject: [PATCH 41/72] Fix maybe-uninitialized warning --- cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh | 2 +- cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh index 8ab6b19b98..75bf41ffd9 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_multi_cta.cuh @@ -377,7 +377,7 @@ __launch_bounds__(BLOCK_SIZE, BLOCK_COUNT) __global__ void search_kernel( const uint32_t min_iteration, \ const uint32_t max_iteration, \ uint32_t* const num_executed_iterations); \ - search_kernel_t kernel; \ + search_kernel_t kernel = nullptr; \ if (result_buffer_size <= 64) { \ SET_MC_KERNEL_1(64) \ } else if (result_buffer_size <= 128) { \ diff --git a/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh b/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh index 219a1dd717..77896bac51 100644 --- a/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh +++ b/cpp/include/raft/neighbors/detail/cagra/search_single_cta.cuh @@ -873,7 +873,7 @@ __launch_bounds__(BLOCK_SIZE, BLOCK_COUNT) __global__ const std::uint32_t hash_bitlen, \ const std::uint32_t small_hash_bitlen, \ const std::uint32_t small_hash_reset_interval); \ - search_kernel_t kernel; \ + search_kernel_t kernel = nullptr; \ if (num_itopk_candidates <= 64) { \ constexpr unsigned max_candidates = 64; \ if (itopk_size <= 64) { \ From 2b4f316b974a23d3a2678d72b7de7d48cc5cbe66 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 12:24:25 -0500 Subject: [PATCH 42/72] FIX remove not used variables --- conda/recipes/libraft/conda_build_config.yaml | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 3435d14a91..24b9992d3a 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -73,41 +73,3 @@ cuda11_cuda_profiler_api_host_version: cuda11_cuda_profiler_api_run_version: - ">=11.4.240,<12" - -# CUDA 12 versions - -libcublas_host_version: - - "=12.0.1.189" - -libcublas_run_version: - - ">=12.0.1.189" - -libcurand_host_version: - - "=10.3.1.50" - -libcurand_run_version: - - ">=10.3.1.50" - -libcusolver_host_version: - - "=11.4.2.57" - -libcusolver_run_version: - - ">=11.4.2.57" - -libcusparse_host_version: - - "=12.0.0.76" - -libcusparse_run_version: - - ">=12.0.0.76" - -cuda_profiler_api_host_version: - - "=12.0.76" - -cuda_profiler_api_run_version: - - ">=12.0.76" - -libnvjitlink_host_version: - - "=12.0.140" - -libnvjitlink_run_version: - - ">=12.0.140" From aec019c6c56189ff58cbe46ee14295c8692e4c4f Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 12:27:04 -0500 Subject: [PATCH 43/72] FIX missing cuda-version specs in libraft recipe --- conda/recipes/libraft/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index e4c0fbcf37..2208b4d4c8 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -153,6 +153,7 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} {% endif %} + - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -215,6 +216,7 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} {% endif %} + - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -280,6 +282,7 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} {% endif %} + - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -341,6 +344,7 @@ outputs: - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} {% endif %} + - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja From 8a8b4adff91868a9621bb686d60f70d314d8b288 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 12:36:50 -0500 Subject: [PATCH 44/72] FIX remove unnecessary CUDA 12 library versions --- conda/recipes/libraft/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 2208b4d4c8..f4ffe78eff 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -64,9 +64,9 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - libnvjitlink {{ libnvjitlink_host_version }} + - cuda-cudart + - cuda-cudart-dev + - libnvjitlink {% endif %} - cuda-version ={{ cuda_version }} - librmm ={{ minor_version }} From 652f88ec42710737fec8c67033a845d73cefef5b Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 12:41:04 -0500 Subject: [PATCH 45/72] FIX cudatoolkit specification fixes --- conda/recipes/libraft/meta.yaml | 4 ++-- conda/recipes/pylibraft/meta.yaml | 3 ++- conda/recipes/raft-dask/meta.yaml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index f4ffe78eff..80a79d0b16 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -291,7 +291,7 @@ outputs: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} {% if cuda_major == "11" %} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -352,7 +352,7 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} {% if cuda_major == "11" %} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 1bf49540fa..2cecc0221c 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -64,7 +64,8 @@ requirements: tests: requirements: - - cudatoolkit ={{ cuda_version }} + - cuda-version ={{ cuda_version } + - cudatoolkit imports: - pylibraft diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 981c079290..c8d164692f 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -74,7 +74,8 @@ requirements: tests: requirements: - - cudatoolkit ={{ cuda_version }} + - cuda-version ={{ cuda_version } + - cudatoolkit imports: - raft_dask From 3790ecb186609813aafd4072104092f837a9017c Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 15:51:19 -0500 Subject: [PATCH 46/72] Apply suggestions from code review Co-authored-by: Bradley Dice --- conda/recipes/libraft/conda_build_config.yaml | 8 +- conda/recipes/libraft/meta.yaml | 75 ++++--------------- conda/recipes/pylibraft/meta.yaml | 16 ++-- conda/recipes/raft-dask/meta.yaml | 16 ++-- dependencies.yaml | 5 -- 5 files changed, 32 insertions(+), 88 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 24b9992d3a..c8dcce90eb 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -34,11 +34,9 @@ h5py_version: nlohmann_json_version: - ">=3.11.2" -# CUDA 11 versions - -# The CTK libraries below are missing from the conda-forge::cudatoolkit -# package. The "*_host_*" version specifiers correspond to `11.8` packages and the -# "*_run_*" version specifiers correspond to `11.x` packages. +# The CTK libraries below are missing from the conda-forge::cudatoolkit package +# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages +# and the "*_run_*" version specifiers correspond to `11.x` packages. cuda11_libcublas_host_version: - "=11.11.3.6" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 80a79d0b16..27b7a5ec31 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -41,9 +41,7 @@ outputs: string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda11') }} {% endif %} - librmm requirements: @@ -53,10 +51,10 @@ outputs: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} + - {{ compiler('cuda') }} - cuda-cudart-dev {{ cuda_version }} {% endif %} + - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -64,7 +62,6 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - cuda-cudart - cuda-cudart-dev - libnvjitlink {% endif %} @@ -74,7 +71,6 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - cuda-cudart - cuda-cudart-dev - libnvjitlink {% endif %} @@ -90,11 +86,8 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - ignore_run_exports_from: {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda11') }} {% endif %} - librmm requirements: @@ -113,16 +106,11 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-cudart - cuda-cudart-dev - cuda-profiler-api - - libcublas - libcublas-dev - - libcurand - libcurand-dev - - libcusolver - libcusolver-dev - - libcusparse - libcusparse-dev - libnvjitlink {% endif %} @@ -139,9 +127,7 @@ outputs: string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda11') }} {% endif %} requirements: build: @@ -149,9 +135,8 @@ outputs: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} + - {{ compiler('cuda') }} + - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} @@ -173,15 +158,10 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart - cuda-cudart-dev - - libcublas - libcublas-dev - - libcurand - libcurand-dev - - libcusolver - libcusolver-dev - - libcusparse - libcusparse-dev {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} @@ -200,11 +180,7 @@ outputs: string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} + - {{ compiler('cuda11') }} {% endif %} requirements: build: @@ -212,9 +188,7 @@ outputs: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} + - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} @@ -236,16 +210,11 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart - cuda-cudart-dev - cuda-cudart-static - - libcublas - libcublas-dev - - libcurand - libcurand-dev - - libcusolver - libcusolver-dev - - libcusparse - libcusparse-dev {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} @@ -268,9 +237,7 @@ outputs: string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda11') }} {% endif %} requirements: build: @@ -278,9 +245,7 @@ outputs: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} + - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} @@ -303,15 +268,10 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart - cuda-cudart-dev - - libcublas - libcublas-dev - - libcurand - libcurand-dev - - libcusolver - libcusolver-dev - - libcusparse - libcusparse-dev {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} @@ -330,9 +290,7 @@ outputs: string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda11') }} {% endif %} requirements: build: @@ -340,9 +298,7 @@ outputs: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} + - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} @@ -364,16 +320,11 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart - cuda-cudart-dev - cuda-cudart-static - - libcublas - libcublas-dev - - libcurand - libcurand-dev - - libcusolver - libcusolver-dev - - libcusparse - libcusparse-dev {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 2cecc0221c..1b3b630798 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -29,18 +29,19 @@ requirements: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda') }} {% endif %} + - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0 + - cuda-python >=11.7.1,<12.0a0 - cudatoolkit - {% else} + {% else %} - cuda-python >=12.0,<13.0a0 - {% endif} + {% endif %} - cuda-version ={{ cuda_version } - cython >=0.29,<0.30 - libraft {{ version }} @@ -52,11 +53,11 @@ requirements: - setuptools run: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0 + - cuda-python >=11.7.1,<12.0a0 - cudatoolkit - {% else} + {% else %} - cuda-python >=12.0,<13.0a0 - {% endif} + {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - libraft {{ version }} - libraft-headers {{ version }} @@ -65,7 +66,6 @@ requirements: tests: requirements: - cuda-version ={{ cuda_version } - - cudatoolkit imports: - pylibraft diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index c8d164692f..477327bec6 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -29,18 +29,19 @@ requirements: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cuda') }} {% endif %} + - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0 + - cuda-python >=11.7.1,<12.0a0 - cudatoolkit - {% else} + {% else %} - cuda-python >=12.0,<13.0a0 - {% endif} + {% endif %} - cython >=0.29,<0.30 - nccl >=2.9.9 - pylibraft {{ version }} @@ -53,11 +54,11 @@ requirements: - ucx-py {{ ucx_py_version }} run: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0 + - cuda-python >=11.7.1,<12.0a0 - cudatoolkit - {% else} + {% else %} - cuda-python >=12.0,<13.0a0 - {% endif} + {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask >=2023.5.1 - dask-core >=2023.5.1 @@ -75,7 +76,6 @@ requirements: tests: requirements: - cuda-version ={{ cuda_version } - - cudatoolkit imports: - raft_dask diff --git a/dependencies.yaml b/dependencies.yaml index 43cd437f2e..f38557ab32 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -169,17 +169,12 @@ dependencies: cuda: "12.0" packages: - cuda-version=12.0 - - cuda-cudart - cuda-cudart-dev - cuda-profiler-api - libcublas-dev - - libcublas - libcurand-dev - - libcurand - libcusolver-dev - - libcusolver - libcusparse-dev - - libcusparse - matrix: cuda: "11.8" packages: From 9dea87df78ba395c7f18f6d02c2134776ed8a31d Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 16:04:45 -0500 Subject: [PATCH 47/72] FIX thrust call --- cpp/include/raft/spectral/detail/matrix_wrappers.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index 07ab1dbeba..ce951a424c 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -111,7 +111,7 @@ class vector_t { buffer_.data(), buffer_.data() + buffer_.size(), value_type{0}, - [] __device__(auto left, auto right) { + [] __host__ __device__(auto left, auto right) { auto abs_left = left > 0 ? left : -left; auto abs_right = right > 0 ? right : -right; return abs_left + abs_right; From cd38bb276fc569fa8b7468aa71c83c8615075c96 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 16:05:32 -0500 Subject: [PATCH 48/72] FIX More review comments applied --- .github/workflows/test.yaml | 2 -- ci/build_cpp.sh | 2 +- .../all_cuda-118_arch-x86_64.yaml | 2 +- .../all_cuda-120_arch-x86_64.yaml | 2 +- dependencies.yaml | 22 +++++++++++++++++-- python/pylibraft/pyproject.toml | 4 ++-- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c1ccb3e14f..081c7e8df3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,8 +39,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: pylibraft - test-before-amd64: "pip install cupy-cuda11x" - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test" wheel-tests-raft-dask: secrets: inherit diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index e4b1d7fafb..853ae095d3 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. set -euo pipefail diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index fe06e21b16..546728d2c6 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,7 +13,7 @@ dependencies: - clang=16.0.1 - cmake>=3.23.1,!=3.25.0 - cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0 +- cuda-python>=11.7.1,<12.0a0 - cuda-version=11.8 - cudatoolkit - cupy>=12.0.0 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 320fec177b..e1b041a1c2 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -15,7 +15,7 @@ dependencies: - cuda-cudart - cuda-cudart-dev - cuda-profiler-api -- cuda-python>=11.7.1,<12.0 +- cuda-python>=12.0,<13.0a0 - cuda-version=12.0 - cupy>=12.0.0 - cxx-compiler diff --git a/dependencies.yaml b/dependencies.yaml index f38557ab32..d52552418b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -135,8 +135,17 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - &cuda_python cuda-python>=11.7.1,<12.0 - &rmm rmm==23.8.* + specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: + cuda: "12.0" + packages: + - cuda-python>=12.0,<13.0a0 + - matrix: # All CUDA 11 versions + packages: + - cuda-python>=11.7.1,<12.0a0 checks: common: - output_types: [conda, requirements] @@ -278,8 +287,17 @@ dependencies: - output_types: [conda, pyproject] packages: - &numpy numpy>=1.21 - - *cuda_python - *rmm + specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: + cuda: "12.0" + packages: + - cuda-python>=12.0,<13.0a0 + - matrix: # All CUDA 11 versions + packages: + - cuda-python>=11.7.1,<12.0a0 run_raft_dask: common: - output_types: [conda, pyproject] diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index ef71065ff4..cb7e7ad8c2 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -16,7 +16,7 @@ requires = [ "cmake>=3.23.1,!=3.25.0", - "cuda-python>=11.7.1,<12.0", + "cuda-python>=11.7.1,<12.0a0", "cython>=0.29,<0.30", "ninja", "rmm==23.8.*", @@ -37,7 +37,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cuda-python>=11.7.1,<12.0", + "cuda-python>=11.7.1,<12.0a0", "numpy>=1.21", "rmm==23.8.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From f4f5a00a87cbc615ba3813dea347fad44e0d72ff Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 16:13:24 -0500 Subject: [PATCH 49/72] FIX rdg precommit --- conda/environments/all_cuda-120_arch-x86_64.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index e1b041a1c2..0ae6154078 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -12,7 +12,6 @@ dependencies: - clang-tools=16.0.1 - clang=16.0.1 - cmake>=3.23.1,!=3.25.0 -- cuda-cudart - cuda-cudart-dev - cuda-profiler-api - cuda-python>=12.0,<13.0a0 @@ -31,13 +30,9 @@ dependencies: - gtest>=1.13.0 - ipython - joblib>=0.11 -- libcublas - libcublas-dev -- libcurand - libcurand-dev -- libcusolver - libcusolver-dev -- libcusparse - libcusparse-dev - nccl>=2.9.9 - ninja From 6452dfb81a0e63a07be92e4cd2163b0be13d9838 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 17:13:23 -0500 Subject: [PATCH 50/72] FIX thrust call further fix --- .../raft/spectral/detail/matrix_wrappers.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index ce951a424c..f3d8aed567 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include #include @@ -107,15 +109,13 @@ class vector_t { value_type nrm1() const { - return thrust::reduce(thrust_policy, - buffer_.data(), - buffer_.data() + buffer_.size(), - value_type{0}, - [] __host__ __device__(auto left, auto right) { - auto abs_left = left > 0 ? left : -left; - auto abs_right = right > 0 ? right : -right; - return abs_left + abs_right; - }); + auto f = cuda::proclaim_return_type([] __device__(auto left, auto right) { + auto abs_left = left > 0 ? left : -left; + auto abs_right = right > 0 ? right : -right; + return abs_left + abs_right; + }); + return thrust::reduce( + thrust_policy, buffer_.data(), buffer_.data() + buffer_.size(), value_type{0}, f); } void fill(value_type value) From 4f27238140474d1a84748a853ff85902e840ffbc Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Tue, 27 Jun 2023 20:07:49 -0500 Subject: [PATCH 51/72] DBG Undo thrust changes to get a clean log --- .../raft/spectral/detail/matrix_wrappers.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index f3d8aed567..07ab1dbeba 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -15,8 +15,6 @@ */ #pragma once -#include - #include #include #include @@ -109,13 +107,15 @@ class vector_t { value_type nrm1() const { - auto f = cuda::proclaim_return_type([] __device__(auto left, auto right) { - auto abs_left = left > 0 ? left : -left; - auto abs_right = right > 0 ? right : -right; - return abs_left + abs_right; - }); - return thrust::reduce( - thrust_policy, buffer_.data(), buffer_.data() + buffer_.size(), value_type{0}, f); + return thrust::reduce(thrust_policy, + buffer_.data(), + buffer_.data() + buffer_.size(), + value_type{0}, + [] __device__(auto left, auto right) { + auto abs_left = left > 0 ? left : -left; + auto abs_right = right > 0 ? right : -right; + return abs_left + abs_right; + }); } void fill(value_type value) From 446cb98e64674c20b6f5f806789354ba86ed3c09 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 28 Jun 2023 09:44:27 -0500 Subject: [PATCH 52/72] FIX remove cudart-static from dependencies of libraft-tests --- conda/recipes/libraft/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 27b7a5ec31..f1dcfb28c0 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -211,7 +211,6 @@ outputs: {% else %} - cuda-profiler-api - cuda-cudart-dev - - cuda-cudart-static - libcublas-dev - libcurand-dev - libcusolver-dev From c99132414ae99d8ef6d59aca28e03f0f0d5cebbf Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 28 Jun 2023 10:53:34 -0500 Subject: [PATCH 53/72] DBG cudart dependency debugging --- conda/recipes/libraft/meta.yaml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index f1dcfb28c0..9a6cb1c8a0 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -52,7 +52,6 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} - - cuda-cudart-dev {{ cuda_version }} {% endif %} - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} @@ -62,7 +61,6 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - cuda-cudart-dev - libnvjitlink {% endif %} - cuda-version ={{ cuda_version }} @@ -71,7 +69,7 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - cuda-cudart-dev + - cuda-cudart - libnvjitlink {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} @@ -106,7 +104,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-cudart-dev + - cuda-cudart - cuda-profiler-api - libcublas-dev - libcurand-dev @@ -136,7 +134,6 @@ outputs: - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} - - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} @@ -158,7 +155,6 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart-dev - libcublas-dev - libcurand-dev - libcusolver-dev @@ -210,7 +206,6 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart-dev - libcublas-dev - libcurand-dev - libcusolver-dev @@ -220,6 +215,12 @@ outputs: - gmock {{ gtest_version }} - gtest {{ gtest_version }} run: + {% if cuda_major == "11" %} + - cudatoolkit + {% else %} + - cuda-cudart + {% endif %} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {{ pin_subpackage('libraft', exact=True) }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} @@ -267,7 +268,6 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart-dev - libcublas-dev - libcurand-dev - libcusolver-dev @@ -319,8 +319,6 @@ outputs: - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - - cuda-cudart-dev - - cuda-cudart-static - libcublas-dev - libcurand-dev - libcusolver-dev From 4498951c98afce8c26be13ade00dc413f82be43f Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 28 Jun 2023 16:10:11 -0500 Subject: [PATCH 54/72] DBG Fix faiss version to pull rapidsai channel package --- conda/recipes/libraft/conda_build_config.yaml | 2 +- conda/recipes/libraft/meta.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index c8dcce90eb..0aea9b20ad 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -26,7 +26,7 @@ glog_version: - ">=0.6.0" faiss_version: - - ">=1.7.1" + - "=1.7.2" h5py_version: - ">=3.8.0" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 9a6cb1c8a0..46fbec3964 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -329,8 +329,8 @@ outputs: - nlohmann_json {{ nlohmann_json_version }} # Temporarily ignore faiss benchmarks on CUDA 12 {% if cuda_major == "11" %} - - libfaiss>=1.7.1 - - faiss-proc=*=cuda + - faiss-proc=*=cuda + - libfaiss {{ faiss_version }} {% endif %} run: - {{ pin_subpackage('libraft', exact=True) }} From b9913a44af9eef135b13815291e8091fe02ef386 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 28 Jun 2023 16:13:59 -0500 Subject: [PATCH 55/72] FIX small cleanup --- conda/recipes/libraft/meta.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 46fbec3964..a58efd6289 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -130,13 +130,13 @@ outputs: requirements: build: - {{ compiler('c') }} + - {{ compiler('cxx') }} {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -181,13 +181,13 @@ outputs: requirements: build: - {{ compiler('c') }} + - {{ compiler('cxx') }} {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -242,13 +242,13 @@ outputs: requirements: build: - {{ compiler('c') }} + - {{ compiler('cxx') }} {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} @@ -294,13 +294,13 @@ outputs: requirements: build: - {{ compiler('c') }} + - {{ compiler('cxx') }} {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} From 821076df3f8e2dac9212ceb9ce2e960c6c9a2407 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 28 Jun 2023 17:37:21 -0500 Subject: [PATCH 56/72] FIX blank space in recipe --- conda/recipes/libraft/conda_build_config.yaml | 2 +- conda/recipes/libraft/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 0aea9b20ad..c8dcce90eb 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -26,7 +26,7 @@ glog_version: - ">=0.6.0" faiss_version: - - "=1.7.2" + - ">=1.7.1" h5py_version: - ">=3.8.0" diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index a58efd6289..0f39d719cb 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -329,7 +329,7 @@ outputs: - nlohmann_json {{ nlohmann_json_version }} # Temporarily ignore faiss benchmarks on CUDA 12 {% if cuda_major == "11" %} - - faiss-proc=*=cuda + - faiss-proc=*=cuda - libfaiss {{ faiss_version }} {% endif %} run: From 127710fed65d0d4628cd8826002c995008f0e63f Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Wed, 28 Jun 2023 21:02:38 -0500 Subject: [PATCH 57/72] FIX missing bracket --- conda/recipes/pylibraft/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 1b3b630798..690ed3ad10 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -42,7 +42,7 @@ requirements: {% else %} - cuda-python >=12.0,<13.0a0 {% endif %} - - cuda-version ={{ cuda_version } + - cuda-version ={{ cuda_version }} - cython >=0.29,<0.30 - libraft {{ version }} - libraft-headers {{ version }} @@ -65,7 +65,7 @@ requirements: tests: requirements: - - cuda-version ={{ cuda_version } + - cuda-version ={{ cuda_version }} imports: - pylibraft From f6aa0737641990ec5dcb4a8b2d8f9bf67af09d21 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 07:15:08 -0500 Subject: [PATCH 58/72] DBG sepcify cuda-version in run reqs of libraft artifact --- conda/recipes/libraft/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 0f39d719cb..8bdb337c0e 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -92,6 +92,7 @@ outputs: run: - {{ pin_subpackage('libraft-headers-only', exact=True) }} - librmm ={{ minor_version }} + - cuda-version {{ cuda_spec }} {% if cuda_major == "11" %} - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} @@ -163,6 +164,7 @@ outputs: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} run: - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-version {{ cuda_spec }} about: home: https://rapids.ai/ license: Apache-2.0 From ac5414ddac5000e1b2cb55dd22aceb9605567aea Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 07:17:08 -0500 Subject: [PATCH 59/72] DBG sepcify cuda-version in run reqs of ann bench artifact --- conda/recipes/libraft/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 8bdb337c0e..c1559fa28d 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -277,6 +277,7 @@ outputs: {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} run: + - cuda-version {{ cuda_spec }} - {{ pin_subpackage('libraft', exact=True) }} about: home: https://rapids.ai/ @@ -336,6 +337,7 @@ outputs: {% endif %} run: - {{ pin_subpackage('libraft', exact=True) }} + - cuda-version {{ cuda_spec }} - glog {{ glog_version }} # Temporarily ignore faiss benchmarks on CUDA 12 {% if cuda_major == "11" %} From a2d84d1641858c21c07dbe9bf83a230bd3844896 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 11:57:40 -0500 Subject: [PATCH 60/72] Apply suggestions from code review Co-authored-by: Bradley Dice --- conda/recipes/libraft/meta.yaml | 18 +++++++++--------- conda/recipes/pylibraft/meta.yaml | 3 --- conda/recipes/raft-dask/meta.yaml | 3 --- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index c1559fa28d..85bc894c89 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -161,10 +161,10 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - cuda-version ={{ cuda_version }} run: - {{ pin_subpackage('libraft-headers', exact=True) }} - - cuda-version {{ cuda_spec }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} about: home: https://rapids.ai/ license: Apache-2.0 @@ -213,7 +213,7 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - cuda-version ={{ cuda_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} run: @@ -275,9 +275,9 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - cuda-version ={{ cuda_version }} run: - - cuda-version {{ cuda_spec }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {{ pin_subpackage('libraft', exact=True) }} about: home: https://rapids.ai/ @@ -327,19 +327,19 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - cuda-version ={{ cuda_version }} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} - # Temporarily ignore faiss benchmarks on CUDA 12 + # Temporarily ignore faiss benchmarks on CUDA 12 because packages do not exist yet {% if cuda_major == "11" %} - faiss-proc=*=cuda - libfaiss {{ faiss_version }} {% endif %} run: - {{ pin_subpackage('libraft', exact=True) }} - - cuda-version {{ cuda_spec }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - glog {{ glog_version }} - # Temporarily ignore faiss benchmarks on CUDA 12 + # Temporarily ignore faiss benchmarks on CUDA 12 because packages do not exist yet {% if cuda_major == "11" %} - faiss-proc=*=cuda - libfaiss {{ faiss_version }} diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 690ed3ad10..2806e94616 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -53,10 +53,7 @@ requirements: - setuptools run: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 - cudatoolkit - {% else %} - - cuda-python >=12.0,<13.0a0 {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - libraft {{ version }} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 477327bec6..b8945fdaf0 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -54,10 +54,7 @@ requirements: - ucx-py {{ ucx_py_version }} run: {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 - cudatoolkit - {% else %} - - cuda-python >=12.0,<13.0a0 {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - dask >=2023.5.1 From 9ea3a4f262eee558c406c62c95a6499c49885e76 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 12:33:20 -0500 Subject: [PATCH 61/72] FIX More review comments applied --- conda/recipes/libraft/meta.yaml | 9 ++------- dependencies.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 85bc894c89..57186e8347 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -61,16 +61,13 @@ outputs: {% if cuda_major == "11" %} - cudatoolkit {% else %} - - libnvjitlink + - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} - librmm ={{ minor_version }} run: {% if cuda_major == "11" %} - cudatoolkit - {% else %} - - cuda-cudart - - libnvjitlink {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - librmm ={{ minor_version }} @@ -111,7 +108,6 @@ outputs: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libnvjitlink {% endif %} about: home: https://rapids.ai/ @@ -207,6 +203,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} + - cuda-cudart-dev - cuda-profiler-api - libcublas-dev - libcurand-dev @@ -219,8 +216,6 @@ outputs: run: {% if cuda_major == "11" %} - cudatoolkit - {% else %} - - cuda-cudart {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {{ pin_subpackage('libraft', exact=True) }} diff --git a/dependencies.yaml b/dependencies.yaml index d52552418b..c536932d22 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -142,10 +142,10 @@ dependencies: - matrix: cuda: "12.0" packages: - - cuda-python>=12.0,<13.0a0 + - &cuda_python12 cuda-python>=12.0,<13.0a0 - matrix: # All CUDA 11 versions packages: - - cuda-python>=11.7.1,<12.0a0 + - &cuda_python11 cuda-python>=11.7.1,<12.0a0 checks: common: - output_types: [conda, requirements] @@ -294,10 +294,10 @@ dependencies: - matrix: cuda: "12.0" packages: - - cuda-python>=12.0,<13.0a0 + - *cuda-python12 - matrix: # All CUDA 11 versions packages: - - cuda-python>=11.7.1,<12.0a0 + - *cuda-python11 run_raft_dask: common: - output_types: [conda, pyproject] From b3a4de8613a344edbc47aa2ade83b296beb34802 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 12:42:12 -0500 Subject: [PATCH 62/72] FIX cuda compiler in conda_build_config --- conda/recipes/pylibraft/conda_build_config.yaml | 3 +++ conda/recipes/raft-dask/conda_build_config.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/conda/recipes/pylibraft/conda_build_config.yaml b/conda/recipes/pylibraft/conda_build_config.yaml index add119d796..41bf15c12c 100644 --- a/conda/recipes/pylibraft/conda_build_config.yaml +++ b/conda/recipes/pylibraft/conda_build_config.yaml @@ -5,6 +5,9 @@ cxx_compiler_version: - 11 cuda_compiler: + - cuda-nvcc + +cuda11_compiler: - nvcc sysroot_version: diff --git a/conda/recipes/raft-dask/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml index 637ca95dcc..fb09c6d1f5 100644 --- a/conda/recipes/raft-dask/conda_build_config.yaml +++ b/conda/recipes/raft-dask/conda_build_config.yaml @@ -5,6 +5,9 @@ cxx_compiler_version: - 11 cuda_compiler: + - cuda-nvcc + +cuda11_compiler: - nvcc sysroot_version: From 47cb0fb1ff9030ecde459b00b9675e8933455f09 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 13:21:47 -0500 Subject: [PATCH 63/72] FIX typo in dependencies --- dependencies.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index c536932d22..200aa52596 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -294,10 +294,10 @@ dependencies: - matrix: cuda: "12.0" packages: - - *cuda-python12 + - *cuda_python12 - matrix: # All CUDA 11 versions packages: - - *cuda-python11 + - *cuda_python11 run_raft_dask: common: - output_types: [conda, pyproject] From 51255e899b0fd36c11ec8fdaa94c09903e63b3c2 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 29 Jun 2023 15:24:39 -0500 Subject: [PATCH 64/72] Apply suggestions from code review --- conda/recipes/raft-dask/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index b8945fdaf0..5b545b73d4 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -27,7 +27,7 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} @@ -72,7 +72,7 @@ requirements: tests: requirements: - - cuda-version ={{ cuda_version } + - cuda-version ={{ cuda_version }} imports: - raft_dask From 0f2549a35a21befe0d98510d2e099659a79fa1f8 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Thu, 29 Jun 2023 16:04:34 -0500 Subject: [PATCH 65/72] Apply suggestions from code review Co-authored-by: Bradley Dice --- conda/recipes/libraft/meta.yaml | 16 +++++++++------- conda/recipes/pylibraft/meta.yaml | 2 +- conda/recipes/raft-dask/meta.yaml | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 57186e8347..af8feda8f9 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -49,7 +49,7 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} @@ -86,10 +86,12 @@ outputs: {% endif %} - librmm requirements: + host: + - cuda-version ={{ cuda_version }} run: - {{ pin_subpackage('libraft-headers-only', exact=True) }} - librmm ={{ minor_version }} - - cuda-version {{ cuda_spec }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} @@ -102,7 +104,7 @@ outputs: - libcusparse {{ cuda11_libcusparse_run_version }} - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-cudart + - cuda-cudart-dev - cuda-profiler-api - libcublas-dev - libcurand-dev @@ -129,7 +131,7 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} @@ -181,7 +183,7 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} @@ -241,7 +243,7 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} @@ -294,7 +296,7 @@ outputs: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 2806e94616..584e1ee497 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -27,7 +27,7 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} {{ cuda_version }} + - {{ compiler('cuda11') }} ={{ cuda_version }} {% else %} - {{ compiler('cuda') }} {% endif %} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 5b545b73d4..872262620f 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -42,6 +42,7 @@ requirements: {% else %} - cuda-python >=12.0,<13.0a0 {% endif %} + - cuda-version ={{ cuda_version }} - cython >=0.29,<0.30 - nccl >=2.9.9 - pylibraft {{ version }} From 4dfaee3d555a818d572c79fef0365e84fc18b99a Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 30 Jun 2023 12:07:27 -0500 Subject: [PATCH 66/72] Apply suggestions from code review Co-authored-by: jakirkham --- conda/recipes/libraft/meta.yaml | 14 +++++++------- conda/recipes/pylibraft/meta.yaml | 1 - conda/recipes/raft-dask/meta.yaml | 1 - 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index af8feda8f9..37c63dcadf 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -58,9 +58,7 @@ outputs: - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} + {% if cuda_major != "11" %} - cuda-cudart-dev {% endif %} - cuda-version ={{ cuda_version }} @@ -142,7 +140,6 @@ outputs: host: - {{ pin_subpackage('libraft-headers', exact=True) }} {% if cuda_major == "11" %} - - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -194,7 +191,6 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} {% if cuda_major == "11" %} - - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -255,7 +251,6 @@ outputs: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} {% if cuda_major == "11" %} - - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -275,6 +270,9 @@ outputs: - cuda-version ={{ cuda_version }} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major == "11" %} + - cudatoolkit + {% endif %} - {{ pin_subpackage('libraft', exact=True) }} about: home: https://rapids.ai/ @@ -307,7 +305,6 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} {% if cuda_major == "11" %} - - cudatoolkit - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} @@ -335,6 +332,9 @@ outputs: run: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major == "11" %} + - cudatoolkit + {% endif %} - glog {{ glog_version }} # Temporarily ignore faiss benchmarks on CUDA 12 because packages do not exist yet {% if cuda_major == "11" %} diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 584e1ee497..975c1d5927 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -38,7 +38,6 @@ requirements: host: {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0a0 - - cudatoolkit {% else %} - cuda-python >=12.0,<13.0a0 {% endif %} diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 872262620f..455dea1a7a 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -38,7 +38,6 @@ requirements: host: {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0a0 - - cudatoolkit {% else %} - cuda-python >=12.0,<13.0a0 {% endif %} From f3c482a02ce4067561fdb48f5071f8ecd6acab63 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 30 Jun 2023 12:48:06 -0500 Subject: [PATCH 67/72] FIX More review comments applied --- conda/recipes/libraft/meta.yaml | 12 ++++++------ conda/recipes/raft-dask/meta.yaml | 4 +--- cpp/bench/ann/CMakeLists.txt | 1 + 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 37c63dcadf..0bde033e8c 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -64,10 +64,10 @@ outputs: - cuda-version ={{ cuda_version }} - librmm ={{ minor_version }} run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - librmm ={{ minor_version }} about: home: https://rapids.ai/ @@ -139,6 +139,7 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - libcublas {{ cuda11_libcublas_host_version }} @@ -156,7 +157,6 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - cuda-version ={{ cuda_version }} run: - {{ pin_subpackage('libraft-headers', exact=True) }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} @@ -190,6 +190,7 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {{ pin_subpackage('libraft', exact=True) }} + - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} @@ -208,14 +209,13 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - cuda-version ={{ cuda_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {{ pin_subpackage('libraft', exact=True) }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} @@ -250,6 +250,7 @@ outputs: host: - {{ pin_subpackage('libraft', exact=True) }} - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} @@ -267,7 +268,6 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - cuda-version ={{ cuda_version }} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} @@ -304,6 +304,7 @@ outputs: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - {{ pin_subpackage('libraft', exact=True) }} + - cuda-version ={{ cuda_version }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} @@ -321,7 +322,6 @@ outputs: - libcusolver-dev - libcusparse-dev {% endif %} - - cuda-version ={{ cuda_version }} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} # Temporarily ignore faiss benchmarks on CUDA 12 because packages do not exist yet diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 455dea1a7a..0f7594cc0b 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -27,9 +27,7 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} + - {{ compiler('cuda11') }} {% endif %} - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 0685d0cc91..ef6698baad 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -28,6 +28,7 @@ option(RAFT_ANN_BENCH_USE_GGNN "Include ggnn algorithm in benchmark" ON) find_package(Threads REQUIRED) # Disable faiss benchmarks on CUDA 12 since faiss is not yet CUDA 12-enabled. +# https://github.com/rapidsai/raft/issues/1627 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.0.0) set(RAFT_ANN_BENCH_USE_FAISS_BFKNN OFF) set(RAFT_ANN_BENCH_USE_FAISS_IVF_FLAT OFF) From a3ba165855b5807d89bb5c8d4a1e914178a5071f Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 30 Jun 2023 14:10:26 -0500 Subject: [PATCH 68/72] Update conda/recipes/raft-dask/meta.yaml Co-authored-by: jakirkham --- conda/recipes/raft-dask/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 0f7594cc0b..79c158d0b2 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -28,6 +28,8 @@ requirements: - {{ compiler('cxx') }} {% if cuda_major == "11" %} - {{ compiler('cuda11') }} + {% else %} + - {{ compiler('cuda') }} {% endif %} - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} From f15d380c60c912a393360c69748d4bc585024f39 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 30 Jun 2023 14:11:36 -0500 Subject: [PATCH 69/72] FIX add ctk back to host of pylibraft and raft-dask --- conda/recipes/pylibraft/meta.yaml | 3 ++- conda/recipes/raft-dask/meta.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 975c1d5927..8157307645 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -38,6 +38,7 @@ requirements: host: {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0a0 + - cudatoolkit {% else %} - cuda-python >=12.0,<13.0a0 {% endif %} @@ -51,10 +52,10 @@ requirements: - scikit-build >=0.13.1 - setuptools run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - libraft {{ version }} - libraft-headers {{ version }} - python x.x diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 79c158d0b2..19b6e20ade 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -38,6 +38,7 @@ requirements: host: {% if cuda_major == "11" %} - cuda-python >=11.7.1,<12.0a0 + - cudatoolkit {% else %} - cuda-python >=12.0,<13.0a0 {% endif %} From fa1d5b45b0cd4ae4acb60b619755102f1b1c6371 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 30 Jun 2023 15:37:59 -0500 Subject: [PATCH 70/72] FIX changes from https://github.com/vyasr/raft/pull/1 --- conda/recipes/pylibraft/meta.yaml | 4 +++- conda/recipes/raft-dask/meta.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml index 8157307645..dd3a96405b 100644 --- a/conda/recipes/pylibraft/meta.yaml +++ b/conda/recipes/pylibraft/meta.yaml @@ -20,7 +20,9 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} + {% endif %} requirements: build: diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index 19b6e20ade..ae814879ef 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -20,7 +20,9 @@ build: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: - - {{ compiler('cuda') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} + {% endif %} requirements: build: From fa45ec304fa525df5344209c78c7b26fabd5c969 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Fri, 30 Jun 2023 16:56:57 -0500 Subject: [PATCH 71/72] FIX remove unnecessary cuda libraries for libraft-tests and libraft-ann-bench --- conda/recipes/libraft/meta.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 0bde033e8c..09ef7ae4ab 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -255,18 +255,9 @@ outputs: - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - libcusolver {{ cuda11_libcusolver_host_version }} - - libcusolver-dev {{ cuda11_libcusolver_host_version }} - - libcusparse {{ cuda11_libcusparse_host_version }} - - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev {% endif %} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} @@ -309,18 +300,9 @@ outputs: - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - libcusolver {{ cuda11_libcusolver_host_version }} - - libcusolver-dev {{ cuda11_libcusolver_host_version }} - - libcusparse {{ cuda11_libcusparse_host_version }} - - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - cuda-profiler-api - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev {% endif %} - glog {{ glog_version }} - nlohmann_json {{ nlohmann_json_version }} From f264b9bef333184b50071c6336f1e3dd163b434f Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Sun, 2 Jul 2023 18:47:10 -0500 Subject: [PATCH 72/72] Update conda/recipes/raft-dask/meta.yaml Co-authored-by: jakirkham --- conda/recipes/raft-dask/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index ae814879ef..b7e933755a 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -29,7 +29,8 @@ requirements: - {{ compiler('c') }} - {{ compiler('cxx') }} {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} + - {{ compiler('cuda11') }} ={{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {% endif %}