diff --git a/CHANGELOG.md b/CHANGELOG.md index 92bc9c9d3..8c7f38da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - PR #32 Python API first pass - PR #37 Python __init__.py package design - PR #38 Add __init__.py empties to resolve issue with PYTHONPATH +- PR #25 Add gpuCI integration ## Improvements diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..22a02c720 --- /dev/null +++ b/build.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# Copyright (c) 2019, NVIDIA CORPORATION. + +# cuSpatial build script + +# This script is used to build the component(s) in this repo from +# source, and can be called with various options to customize the +# build as needed (see the help output for details) + +# Abort script on first error +set -e + +NUMARGS=$# +ARGS=$* + +# NOTE: ensure all dir changes are relative to the location of this +# script, and that this script resides in the repo dir! +REPODIR=$(cd $(dirname $0); pwd) + +VALIDARGS="clean libcuspatial cuspatial -v -g -n -h" +HELP="$0 [clean] [libcuspatial] [cuspatial] [-v] [-g] [-n] [-h] + clean - remove all existing build artifacts and configuration (start + over) + libcuspatial - build the libcuspatial C++ code only + cuspatial - build the cuspatial Python package + -v - verbose build mode + -g - build for debug + -n - no install step + -h - print this text + default action (no args) is to build and install 'libcuspatial' then + 'cuspatial' targets +" +LIBCUSPATIAL_BUILD_DIR=${REPODIR}/cpp/build +CUSPATIAL_BUILD_DIR=${REPODIR}/python/cuspatial/build +BUILD_DIRS="${LIBCUSPATIAL_BUILD_DIR} ${CUSPATIAL_BUILD_DIR}" + +# Set defaults for vars modified by flags to this script +VERBOSE="" +BUILD_TYPE=Release +INSTALL_TARGET=install + +# Set defaults for vars that may not have been defined externally +# FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check +# CONDA_PREFIX, but there is no fallback from there! +INSTALL_PREFIX=${INSTALL_PREFIX:=${PREFIX:=${CONDA_PREFIX}}} +PARALLEL_LEVEL=${PARALLEL_LEVEL:=""} + +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +if hasArg -h; then + echo "${HELP}" + exit 0 +fi + +# Check for valid usage +if (( ${NUMARGS} != 0 )); then + for a in ${ARGS}; do + if ! (echo " ${VALIDARGS} " | grep -q " ${a} "); then + echo "Invalid option: ${a}" + exit 1 + fi + done +fi + +# Process flags +if hasArg -v; then + VERBOSE=1 +fi +if hasArg -g; then + BUILD_TYPE=Debug +fi +if hasArg -n; then + INSTALL_TARGET="" +fi + +# If clean given, run it prior to any other steps +if hasArg clean; then + # If the dirs to clean are mounted dirs in a container, the + # contents should be removed but the mounted dirs will remain. + # The find removes all contents but leaves the dirs, the rmdir + # attempts to remove the dirs but can fail safely. + for bd in ${BUILD_DIRS}; do + if [ -d ${bd} ]; then + find ${bd} -mindepth 1 -delete + rmdir ${bd} || true + fi + done +fi + +################################################################################ +# Configure, build, and install libcuspatial +if (( ${NUMARGS} == 0 )) || hasArg libcuspatial; then + + mkdir -p ${LIBCUSPATIAL_BUILD_DIR} + cd ${LIBCUSPATIAL_BUILD_DIR} + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DCMAKE_CXX11_ABI=ON \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} .. + make -j ${PARALLEL_LEVEL} install VERBOSE=${VERBOSE} +fi + +# Build and install the cuspatial Python package +if (( ${NUMARGS} == 0 )) || hasArg cuspatial; then + + cd ${REPODIR}/python/cuspatial + if [[ ${INSTALL_TARGET} != "" ]]; then + python setup.py build_ext --inplace + python setup.py install --single-version-externally-managed --record=record.txt + else + python setup.py build_ext --inplace --library-dir=${LIBCUSPATIAL_BUILD_DIR} + fi +fi + diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh new file mode 100644 index 000000000..565ee2848 --- /dev/null +++ b/ci/checks/changelog.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright (c) 2018, NVIDIA CORPORATION. +######################### +# cuSpatial CHANGELOG Tester # +######################### + +# Checkout master for comparison +git checkout --quiet master + +# Switch back to tip of PR branch +git checkout --quiet current-pr-branch + +# Ignore errors during searching +set +e + +# Get list of modified files between matster and PR branch +CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md` +# Check if CHANGELOG has PR ID +PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` +RETVAL=0 + +# Return status of check result +if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then + echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" +else + echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" + RETVAL=1 +fi + +cat << EOF + It should be placed under the section for the appropriate release. + It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. + It should be formatted as '- PR # ' + Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' + + +EOF + +exit $RETVAL + diff --git a/ci/checks/style.sh b/ci/checks/style.sh new file mode 100644 index 000000000..2d371f812 --- /dev/null +++ b/ci/checks/style.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright (c) 2018, NVIDIA CORPORATION. +##################### +# cuSpatial Style Tester # +##################### + +# Ignore errors and set path +set +e +PATH=/conda/bin:$PATH +LC_ALL=C.UTF-8 +LANG=C.UTF-8 + +# Activate common conda env +source activate gdf + +# Run isort and get results/return code +ISORT=`isort --recursive --check-only python` +ISORT_RETVAL=$? + +# Run black and get results/return code +BLACK=`black --check python` +BLACK_RETVAL=$? + +# Run flake8 and get results/return code +FLAKE=`flake8 python` +FLAKE_RETVAL=$? + +# Run flake8-cython and get results/return code +FLAKE_CYTHON=`flake8 --config=python/cuspatial/.flake8.cython` +FLAKE_CYTHON_RETVAL=$? + +# Output results if failure otherwise show pass +if [ "$ISORT_RETVAL" != "0" ]; then + echo -e "\n\n>>>> FAILED: isort style check; begin output\n\n" + echo -e "$ISORT" + echo -e "\n\n>>>> FAILED: isort style check; end output\n\n" +else + echo -e "\n\n>>>> PASSED: isort style check\n\n" +fi + diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh new file mode 100644 index 000000000..8558d6a9a --- /dev/null +++ b/ci/cpu/build.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (c) 2018, NVIDIA CORPORATION. +###################################### +# cuSpatial CPU conda build script for CI # +###################################### +set -e + +# Logger function for build status output +function logger() { + echo -e "\n>>>> $@\n" +} + +# Set path and build parallel level +export PATH=/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=4 +export CUDF_HOME="${WORKSPACE}/cudf" + +# Set home to the job's workspace +export HOME=$WORKSPACE + +# Switch to project root; also root of repo checkout +cd $WORKSPACE + +# Get latest tag and number of commits since tag +export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags` +export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count` +export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` + +################################################################################ +# SETUP - Check environment +################################################################################ + +logger "Get env..." +env + +logger "Activate conda env..." +source activate gdf + +logger "Check versions..." +python --version +gcc --version +g++ --version +conda list + +# FIX Added to deal with Anancoda SSL verification issues during conda builds +conda config --set ssl_verify False + +########################################################################################## +# BUILD - Conda package builds (conda deps: libcupatial <- cuspatial) +########################################################################################## + +logger "Clone cudf" +git clone https://github.com/rapidsai/cudf.git -b branch-$MINOR_VERSION $CUDF_HOME +cd $CUDF_HOME +git submodule update --init --remote --recursive + +logger "Build conda pkg for libcuspatial..." +cd $WORKSPACE +source ci/cpu/libcuspatial/build_libcuspatial.sh + +logger "Build conda pkg for cuspatial..." +source ci/cpu/cuspatial/build_cuspatial.sh + +################################################################################ +# UPLOAD - Conda packages +################################################################################ + +logger "Upload conda pkgs..." +source ci/cpu/upload_anaconda.sh + diff --git a/ci/cpu/cuspatial/build_cuspatial.sh b/ci/cpu/cuspatial/build_cuspatial.sh new file mode 100644 index 000000000..15f833f9c --- /dev/null +++ b/ci/cpu/cuspatial/build_cuspatial.sh @@ -0,0 +1,5 @@ +set -e + +echo "Building cuspatial" +conda build conda/recipes/cuspatial --python=$PYTHON + diff --git a/ci/cpu/libcuspatial/build_libcuspatial.sh b/ci/cpu/libcuspatial/build_libcuspatial.sh new file mode 100644 index 000000000..f4a395746 --- /dev/null +++ b/ci/cpu/libcuspatial/build_libcuspatial.sh @@ -0,0 +1,5 @@ +set -e + +echo "Building libcuspatial" +conda build conda/recipes/libcuspatial + diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh new file mode 100644 index 000000000..158d99fb1 --- /dev/null +++ b/ci/cpu/prebuild.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +#Upload cuspatial once per PYTHON +if [[ "$CUDA" == "9.2" ]]; then + export UPLOAD_CUSPATIAL=1 +else + export UPLOAD_CUSPATIAL=0 +fi + +#Upload libcuspatial once per CUDA +if [[ "$PYTHON" == "3.6" ]]; then + export UPLOAD_LIBCUSPATIAL=1 +else + export UPLOAD_LIBCUSPATIAL=0 +fi + diff --git a/ci/cpu/upload_anaconda.sh b/ci/cpu/upload_anaconda.sh new file mode 100644 index 000000000..849d665d9 --- /dev/null +++ b/ci/cpu/upload_anaconda.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +export LIBCUSPATIAL_FILE=`conda build conda/recipes/libcuspatial --output` +export CUSPATIAL_FILE=`conda build conda/recipes/cuspatial --python=$PYTHON --output` + +SOURCE_BRANCH=master +CUDA_REL=${CUDA_VERSION%.*} + +# Restrict uploads to master branch +if [ ${GIT_BRANCH} != ${SOURCE_BRANCH} ]; then + echo "Skipping upload" + return 0 +fi + +if [ -z "$MY_UPLOAD_KEY" ]; then + echo "No upload key" + return 0 +fi + +if [ "$UPLOAD_LIBCUSPATIAL" == "1" ]; then + LABEL_OPTION="--label main --label cuda${CUDA_REL}" + echo "LABEL_OPTION=${LABEL_OPTION}" + + test -e ${LIBCUSPATIAL_FILE} + echo "Upload libcuspatial" + echo ${LIBCUSPATIAL_FILE} + anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${LIBCUSPATIAL_FILE} +fi + +if [ "$UPLOAD_CUSPATIAL" == "1" ]; then + LABEL_OPTION="--label main --label cuda9.2 --label cuda10.0" + echo "LABEL_OPTION=${LABEL_OPTION}" + + test -e ${CUSPATIAL_FILE} + echo "Upload cuspatial" + echo ${CUSPATIAL_FILE} + anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${CUSPATIAL_FILE} +fi + diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh new file mode 100644 index 000000000..e14c9d55d --- /dev/null +++ b/ci/gpu/build.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Copyright (c) 2018, NVIDIA CORPORATION. +######################################### +# cuSpatial GPU build and test script for CI # +######################################### +set -e +NUMARGS=$# +ARGS=$* + +# Logger function for build status output +function logger() { + echo -e "\n>>>> $@\n" +} + +# Arg parsing function +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +# Set path and build parallel level +export PATH=/conda/bin:/usr/local/cuda/bin:$PATH +export PARALLEL_LEVEL=4 +export CUDA_REL=${CUDA_VERSION%.*} +export CUDF_HOME="${WORKSPACE}/cudf" + +# Set home to the job's workspace +export HOME=$WORKSPACE + +# Parse git describe +cd $WORKSPACE +export GIT_DESCRIBE_TAG=`git describe --tags` +export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` + +################################################################################ +# SETUP - Check environment +################################################################################ + +logger "Check environment..." +env + +logger "Check GPU usage..." +nvidia-smi + +logger "Activate conda env..." +source activate gdf +conda install "cudf=${MINOR_VERSION}.*" "cudatoolkit=$CUDA_REL" + +logger "Check versions..." +python --version +$CC --version +$CXX --version +conda list + +################################################################################ +# cuSpatial currently requires a the cudf repo for private headers +################################################################################ + +logger "Clone cudf" +git clone https://github.com/rapidsai/cudf.git -b branch-$MINOR_VERSION ${CUDF_HOME} +cd $CUDF_HOME +git submodule update --init --remote --recursive + +################################################################################ +# BUILD - Build libcuspatial and cuSpatial from source +################################################################################ + +logger "Build cuSpatial" +cd $WORKSPACE +./build.sh clean libcuspatial cuspatial + +############################################################################### +# TEST - Run libcuspatial and cuSpatial Unit Tests +############################################################################### + +if hasArg --skip-tests; then + logger "Skipping tests..." +else + logger "Download/Generate Test Data" + #TODO + + logger "Test cuSpatial" + #TODO + + #Python Unit tests for cuSpatial + cd $WORKSPACE/python/cuspatial + py.test --cache-clear --junitxml=${WORKSPACE}/junit-cuspatial.xml -v +fi + diff --git a/conda/recipes/cuspatial/build.sh b/conda/recipes/cuspatial/build.sh new file mode 100644 index 000000000..e1dbcfd17 --- /dev/null +++ b/conda/recipes/cuspatial/build.sh @@ -0,0 +1,5 @@ + +# Copyright (c) 2018-2019, NVIDIA CORPORATION. + +# This assumes the script is executed from the root of the repo directory +./build.sh cuspatial diff --git a/conda/recipes/cuspatial/meta.yaml b/conda/recipes/cuspatial/meta.yaml new file mode 100644 index 000000000..9cf234a2f --- /dev/null +++ b/conda/recipes/cuspatial/meta.yaml @@ -0,0 +1,43 @@ +# Copyright (c) 2018, NVIDIA CORPORATION. + +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %} +{% set py_version=environ.get('CONDA_PY', 36) %} + +package: + name: cuspatial + version: {{ version }} + +source: + path: ../../.. + +build: + number: {{ git_revision_count }} + string: py{{ py_version }}_{{ git_revision_count }} + +requirements: + build: + - python + - cython >=0.29,<0.30 + - setuptools + - cudf {{ minor_version }}.* + - libcuspatial {{ minor_version}}.* + + run: + - python + - cudf {{ minor_version }}.* + - libcuspatial {{ minor_version}}.* + - cython >=0.29,<0.30 + +test: + commands: + - python -c "import cuspatial" + +about: + home: http://rapids.ai/ + license: Apache-2.0 + license_family: Apache + license_file: LICENSE + summary: cuSpatial GPU Spatial and Trajectory Data Management and Analytics Library + diff --git a/conda/recipes/libcuspatial/build.sh b/conda/recipes/libcuspatial/build.sh new file mode 100644 index 000000000..02754811b --- /dev/null +++ b/conda/recipes/libcuspatial/build.sh @@ -0,0 +1,8 @@ +# Copyright (c) 2018-2019, NVIDIA CORPORATION. + +# This assumes the script is executed from the root of the repo directory +# show environment +printenv +# build cuspatial with verbose output +cd $WORKSPACE +./build.sh -v libcuspatial diff --git a/conda/recipes/libcuspatial/meta.yaml b/conda/recipes/libcuspatial/meta.yaml new file mode 100644 index 000000000..23d768909 --- /dev/null +++ b/conda/recipes/libcuspatial/meta.yaml @@ -0,0 +1,47 @@ +# Copyright (c) 2018, NVIDIA CORPORATION. + +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set git_revision_count=environ.get('GIT_DESCRIBE_NUMBER', 0) %} +{% set cuda_version='.'.join(environ.get('CUDA_VERSION', '9.2').split('.')[:2]) %} + +package: + name: libcuspatial + version: {{ version }} + +source: + path: ../../.. + +build: + number: {{ git_revision_count }} + string: cuda{{ cuda_version }}_{{ git_revision_count }} + script_env: + - CC + - CXX + - CUDAHOSTCXX + - PARALLEL_LEVEL + - CUDF_HOME + run_exports: + - {{ pin_subpackage("libcuspatial", max_pin="x.x") }} + +requirements: + build: + - cmake >=3.12.4 + host: + - libcudf {{ minor_version }}.* + - cudatoolkit {{ cuda_version }}.* + run: + - {{ pin_compatible('cudatoolkit', max_pin='x.x') }} + - libcudf {{ minor_version }}.* + +test: + commands: + - test -f $PREFIX/lib/libcuspatial.so + +about: + home: http://rapids.ai/ + license: Apache-2.0 + license_family: Apache + license_file: LICENSE + summary: libcuspatial library +