From 726b7c14800e93ec3af959911bb43593be6c014b Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 7 Jul 2021 14:17:21 -0400 Subject: [PATCH 1/7] Fill in build script --- continuous_integration/gpuci/build.sh | 63 ++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index f87f5c14cbb..abe2fc0734a 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -1 +1,62 @@ -# TODO \ No newline at end of file +############################################## +# Dask GPU build and test script for CI # +############################################## +set -e +NUMARGS=$# +ARGS=$* + +# Arg parsing function +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +# Set path and build parallel level +export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} + +# Set home to the job's workspace +export HOME="$WORKSPACE" + +# Switch to project root; also root of repo checkout +cd "$WORKSPACE" + +# Determine CUDA release version +export CUDA_REL=${CUDA_VERSION%.*} + +################################################################################ +# SETUP - Check environment +################################################################################ + +gpuci_logger "Check environment variables" +env + +gpuci_logger "Check GPU usage" +nvidia-smi + +gpuci_logger "Activate conda env" +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids + +gpuci_logger "Install dask dependencies" +gpuci_conda_retry env update -y --name rapids --file "$WORKSPACE/continuous_integration/environment-3.8.yaml" + +gpuci_logger "Install testing dependencies" +gpuci_conda_retry install -y \ + cudf=21.08 \ + cupy + +gpuci_logger "Install dask" +python setup.py install + +gpuci_logger "Check compiler versions" +python --version +$CC --version +$CXX --version + +gpuci_logger "Check conda environment" +conda info +conda config --show-sources +conda list --show-channel-urls + +gpuci_logger "Python py.test for dask" +py.test $WORKSPACE -n 4 --junitxml="$WORKSPACE/junit-dask.xml" -v --cov-config="$WORKSPACE/.coveragerc" --cov=dask --cov-report=xml:"$WORKSPACE/dask-coverage.xml" --cov-report term From 1c4e42ef998eaf3e13ab4936588b9d3f2f9c7537 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 7 Jul 2021 14:38:45 -0400 Subject: [PATCH 2/7] Add matrix yaml --- continuous_integration/gpuci/axis.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 continuous_integration/gpuci/axis.yaml diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml new file mode 100644 index 00000000000..bfb01329d31 --- /dev/null +++ b/continuous_integration/gpuci/axis.yaml @@ -0,0 +1,10 @@ +PYTHON_VER: +- 3.8 + +CUDA_VER: +- 11.2 + +LINUX_VER: +- ubuntu18.04 + +excludes: \ No newline at end of file From 59c56c4cd2bedf3f65551a4d88afccf7f523285c Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:58:35 -0400 Subject: [PATCH 3/7] env update doesn't take confirmation flag --- continuous_integration/gpuci/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index abe2fc0734a..5e15f19c6fc 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -38,7 +38,7 @@ gpuci_logger "Activate conda env" conda activate rapids gpuci_logger "Install dask dependencies" -gpuci_conda_retry env update -y --name rapids --file "$WORKSPACE/continuous_integration/environment-3.8.yaml" +gpuci_conda_retry env update --name rapids --file "$WORKSPACE/continuous_integration/environment-3.8.yaml" gpuci_logger "Install testing dependencies" gpuci_conda_retry install -y \ From a4122dcce21a8772110a04f6c475756aa382890c Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 23 Jul 2021 15:25:03 -0400 Subject: [PATCH 4/7] Try to speed up testing dep install --- continuous_integration/gpuci/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index 5e15f19c6fc..4908906b610 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -41,9 +41,10 @@ gpuci_logger "Install dask dependencies" gpuci_conda_retry env update --name rapids --file "$WORKSPACE/continuous_integration/environment-3.8.yaml" gpuci_logger "Install testing dependencies" -gpuci_conda_retry install -y \ +gpuci_mamba_retry install -y \ cudf=21.08 \ - cupy + cupy \ + cudatoolkit=$CUDA_REL gpuci_logger "Install dask" python setup.py install From 8455986c1930442686b8b3418a7d23f140027c64 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 23 Jul 2021 16:10:30 -0400 Subject: [PATCH 5/7] Only run tests marked for GPU --- continuous_integration/gpuci/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index 4908906b610..a0cedd97d89 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -60,4 +60,4 @@ conda config --show-sources conda list --show-channel-urls gpuci_logger "Python py.test for dask" -py.test $WORKSPACE -n 4 --junitxml="$WORKSPACE/junit-dask.xml" -v --cov-config="$WORKSPACE/.coveragerc" --cov=dask --cov-report=xml:"$WORKSPACE/dask-coverage.xml" --cov-report term +py.test $WORKSPACE -n 4 -v -m gpu --junitxml="$WORKSPACE/junit-dask.xml" --cov-config="$WORKSPACE/.coveragerc" --cov=dask --cov-report=xml:"$WORKSPACE/dask-coverage.xml" --cov-report term From 7a5d7a4fd81ad5ce0d488b5ad48a3c91199082aa Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 26 Jul 2021 15:16:37 -0400 Subject: [PATCH 6/7] Change script to reflect new Dask build image --- continuous_integration/gpuci/axis.yaml | 3 +++ continuous_integration/gpuci/build.sh | 11 +---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml index bfb01329d31..76f19ff492c 100644 --- a/continuous_integration/gpuci/axis.yaml +++ b/continuous_integration/gpuci/axis.yaml @@ -7,4 +7,7 @@ CUDA_VER: LINUX_VER: - ubuntu18.04 +RAPIDS_VER: +- 21.08 + excludes: \ No newline at end of file diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index a0cedd97d89..46ea4a4d550 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -35,16 +35,7 @@ nvidia-smi gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -gpuci_logger "Install dask dependencies" -gpuci_conda_retry env update --name rapids --file "$WORKSPACE/continuous_integration/environment-3.8.yaml" - -gpuci_logger "Install testing dependencies" -gpuci_mamba_retry install -y \ - cudf=21.08 \ - cupy \ - cudatoolkit=$CUDA_REL +conda activate dask gpuci_logger "Install dask" python setup.py install From a3fe896a43c4629e8b5a61b0e6a9118ac7676018 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 26 Jul 2021 18:17:44 -0400 Subject: [PATCH 7/7] Remove compiler check --- continuous_integration/gpuci/build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh index 46ea4a4d550..6eedac6629e 100644 --- a/continuous_integration/gpuci/build.sh +++ b/continuous_integration/gpuci/build.sh @@ -42,8 +42,6 @@ python setup.py install gpuci_logger "Check compiler versions" python --version -$CC --version -$CXX --version gpuci_logger "Check conda environment" conda info