Merge pull request #3013 from rapidsai/branch-0.16

[gpuCI] Auto-merge branch-0.16 to branch-0.17 [skip ci]

CHANGELOG.md

@@ -68,6 +68,7 @@
 - PR #2930: Pin libfaiss to <=1.6.3
 - PR #2928: Updating Estimators Derived from Base for Consistency
 - PR #2942: Adding `cuml.experimental` to the Docs
+- PR #3010: Improve gpuCI Scripts
 
 ## Bug Fixes
 - PR #2973: Allow data imputation for nan values

ci/cpu/build.sh

@@ -1,27 +1,24 @@
 #!/bin/bash
 # Copyright (c) 2018, NVIDIA CORPORATION.
-######################################
-# cuML CPU conda build script for CI #
-######################################
+##############################################
+# cuML CPU conda build script for CI         # 
+##############################################
 set -ex
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
-
-# Set versions of packages needed to be grabbed
-export CUDF_VERSION=0.8.*
-export NVSTRINGS_VERSION=0.8.*
-export RMM_VERSION=0.8.*
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
 
+# Determine CUDA release version
+export CUDA_REL=${CUDA_VERSION%.*}
+
+ # Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 # Switch to project root; also root of repo checkout
 cd $WORKSPACE
 
@@ -34,17 +31,22 @@ fi
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Check environment variables"
 env
 
-logger "Activate conda env..."
-source activate gdf
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check compiler versions"
 python --version
-gcc --version
-g++ --version
-conda list
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
@@ -53,18 +55,32 @@ conda config --set ssl_verify False
 # BUILD - Conda package builds (conda deps: libcuml <- cuml)
 ################################################################################
 
-logger "Build conda pkg for libcuml..."
-source ci/cpu/libcuml/build_libcuml.sh
+if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+  if [ "$BUILD_LIBCUML" == '1' -o "$BUILD_CUML" == '1' ]; then
+    gpuci_logger "Build conda pkg for libcuml"
+    gpuci_conda_retry build conda/recipes/libcuml
+  fi
+else
+  if [ "$BUILD_LIBCUML" == '1' ]; then
+    gpuci_logger "PROJECT FLASH: Build conda pkg for libcuml"
+    gpuci_conda_retry build conda/recipes/libcuml --dirty --no-remove-work-dir
+  fi
+fi
 
-logger "Build conda pkg for cuml..."
-source ci/cpu/cuml/build_cuml.sh
+if [ "$BUILD_CUML" == '1' ]; then
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_logger "Build conda pkg for cuml"
+    gpuci_conda_retry build conda/recipes/cuml --python=${PYTHON}
+  else
+    gpuci_logger "PROJECT FLASH: Build conda pkg for cuml"
+    gpuci_conda_retry build -c ci/artifacts/cuml/cpu/conda-bld/ --dirty --no-remove-work-dir conda/recipes/cuml --python=${PYTHON}
+  fi
+fi
 
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
-logger "Upload conda pkgs for libcuml..."
-source ci/cpu/libcuml/upload-anaconda.sh
+gpuci_logger "Upload conda pkgs"
+source ci/cpu/upload.sh
 
-logger "Upload conda pkg for cuml..."
-source ci/cpu/cuml/upload-anaconda.sh

ci/cpu/upload.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+# Setup 'gpuci_retry' for upload retries (results in 4 total attempts)
+export GPUCI_RETRY_MAX=3
+export GPUCI_RETRY_SLEEP=30
+
+# Set default label options if they are not defined elsewhere
+export LABEL_OPTION=${LABEL_OPTION:-"--label main"}
+
+# Skip uploads unless BUILD_MODE == "branch"
+if [ ${BUILD_MODE} != "branch" ]; then
+  echo "Skipping upload"
+  return 0
+fi
+
+# Skip uploads if there is no upload key
+if [ -z "$MY_UPLOAD_KEY" ]; then
+  echo "No upload key"
+  return 0
+fi
+
+################################################################################
+# SETUP - Get conda file output locations
+################################################################################
+
+gpuci_logger "Get conda file output locations"
+
+export LIBCUML_FILE=`conda build conda/recipes/libcuml --output`
+export CUML_FILE=`conda build conda/recipes/cuml --python=$PYTHON --output`
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+gpuci_logger "Starting conda uploads"
+
+if [ "$BUILD_LIBCUML" == "1" && "$UPLOAD_LIBCUML" == "1" ]; then
+  test -e ${LIBCUML_FILE}
+  echo "Upload libcuml"
+  echo ${LIBCUML_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUML_FILE}
+fi
+
+if [ "$BUILD_CUML" == "1" && "$UPLOAD_CUML" == "1" ]; then
+  test -e ${CUML_FILE}
+  echo "Upload cuml"
+  echo ${CUML_FILE}
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUML_FILE}
+fi
+

ci/docs/build.sh

@@ -18,32 +18,39 @@ export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 export NIGHTLY_VERSION=$(echo $BRANCH_VERSION | awk -F. '{print $2}')
 export PROJECTS=(cuml libcuml)
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
+
 # TODO: Move installs to docs-build-env meta package
 conda install -c anaconda beautifulsoup4 jq
 pip install sphinx-markdown-tables
 
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
-conda list
+
+gpuci_logger "Show conda info"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # Build Doxygen docs
-logger "Build Doxygen docs..."
+gpuci_logger "Build Doxygen docs"
 cd $PROJECT_WORKSPACE/cpp/build
 make doc
 
 # Build Python docs
-logger "Build Sphinx docs..."
+gpuci_logger "Build Sphinx docs"
 cd $PROJECT_WORKSPACE/docs
 make html
 
@@ -54,7 +61,7 @@ for PROJECT in ${PROJECTS[@]}; do
     if [ ! -d "api/$PROJECT/$BRANCH_VERSION" ]; then
         mkdir -p api/$PROJECT/$BRANCH_VERSION
     fi
-    rm -rf $DOCS_WORKSPACE/api/$PROJECT/$BRANCH_VERSION/*	
+    rm -rf $DOCS_WORKSPACE/api/$PROJECT/$BRANCH_VERSION/*
 done