diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 55069b0f53..23a719f27f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,27 +68,23 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cuml: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: cuml - package-dir: python - + script: ci/build_wheel.sh # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use # the clone as is. extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - - skbuild-configure-options: "-DCUML_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DDISABLE_DEPRECATION_WARNINGS=ON -DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/python/cumlprims_mg/" wheel-publish-cuml: needs: wheel-build-cuml secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 894d88a223..b12dced1d7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -95,26 +95,17 @@ jobs: wheel-build-cuml: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08 with: build_type: pull-request - package-name: cuml - package-dir: python + script: ci/build_wheel.sh extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DCUML_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DDISABLE_DEPRECATION_WARNINGS=ON -DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/python/cumlprims_mg/" wheel-tests-cuml: needs: wheel-build-cuml secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: pull-request - package-name: cuml - # Always want to test against latest dask/distributed. - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # On arm also need to install CMake because treelite needs to be compiled (no wheels available for arm). - test-before-arm64: "pip install cmake && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # parallelization is based on current test memory usage - test-unittest: "python -m pytest ./python/cuml/tests -k 'not test_sparse_pca_inputs' -n 8 --ignore=python/cuml/tests/dask && python -m pytest ./python/cuml/tests -k 'test_sparse_pca_inputs' && python -m pytest ./python/cuml/tests/dask" - test-smoketest: "python ci/wheel_smoke_test_cuml.py" + script: ci/test_wheel.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c460ac3fcd..74314aaa36 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,15 +42,10 @@ jobs: test_script: "ci/test_python_dask.sh" wheel-tests-cuml: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: cuml - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # On arm also need to install CMake because treelite needs to be compiled (no wheels available for arm). - test-before-arm64: "pip install cmake && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # parallelization is based on current test memory usage - test-unittest: "python -m pytest ./python/cuml/tests -k 'not test_sparse_pca_inputs' -n 8 --ignore=python/cuml/tests/dask && python -m pytest ./python/cuml/tests -k 'test_sparse_pca_inputs' && python -m pytest ./python/cuml/tests/dask" + script: ci/test_wheel.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 0000000000..979a17014b --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff + +cd python/ + +SKBUILD_CONFIGURE_OPTIONS="-DCUML_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DDISABLE_DEPRECATION_WARNINGS=ON -DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/" \ + python -m pip wheel . \ + -w dist \ + -vvv \ + --no-deps \ + --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100755 index 0000000000..eef8e831c5 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +mkdir -p ./dist +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# On arm also need to install CMake because treelite needs to be compiled (no wheels available for arm). +if [[ "$(arch)" == "aarch64" ]]; then + python -m pip install cmake +fi + +# Always install latest dask for testing +python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08 + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/cuml*.whl)[test] + +# Run smoke tests for aarch64 pull requests +if [[ "$(arch)" == "aarch64" && "${RAPIDS_BUILD_TYPE}" == "pull-request" ]]; then + python ci/wheel_smoke_test_cuml.py +else + python -m pytest ./python/cuml/tests -k 'not test_sparse_pca_inputs' -n 8 --ignore=python/cuml/tests/dask && python -m pytest ./python/cuml/tests -k 'test_sparse_pca_inputs' && python -m pytest ./python/cuml/tests/dask +fi