sync with upstream

CHANGELOG.md

@@ -1,6 +1,9 @@
 # cuML 0.15.0 (Date TBD)
 
 ## New Features
+- PR #2554: Hashing Vectorizer and general vectorizer improvements
+- PR #2240: Making Dask models pickleable
+- PR #2267: CountVectorizer estimator
 - PR #2261: Exposing new FAISS metrics through Python API
 - PR #2287: Single-GPU TfidfTransformer implementation
 - PR #2289: QR SVD solver for MNMG PCA
@@ -9,6 +12,11 @@
 - PR #2394: Adding cosine & correlation distance for KNN
 - PR #2392: PCA can accept sparse inputs, and sparse prim for computing covariance
 - PR #2465: Support pandas 1.0+
+- PR #2519: Precision recall curve using cupy
+- PR #2500: Replace UMAP functionality dependency on nvgraph with RAFT Spectral Clustering
+- PR #2520: TfidfVectorizer estimator
+- PR #2211: MNMG KNN Classifier & Regressor
+- PR #2461: Add KNN Sparse Output Functionality
 
 ## Improvements
 - PR #2336: Eliminate `rmm.device_array` usage
@@ -19,6 +27,7 @@
 - PR #2308: Using fixture for Dask client to eliminate possiblity of not closing
 - PR #2345: make C++ logger level definition to be the same as python layer
 - PR #2329: Add short commit hash to conda package name
+- PR #2362: Implement binary/multi-classification log loss with cupy
 - PR #2363: Update threshold and make other changes for stress tests
 - PR #2371: Updating MBSGD tests to use larger batches
 - PR #2380: Pinning libcumlprims version to ease future updates
@@ -35,11 +44,33 @@
 - PR #2453: Add CumlArray to API doc
 - PR #2440: Use Treelite Conda package
 - PR #2403: Support for input and output type consistency in logistic regression predict_proba
+- PR #2473: Add metrics.roc_auc_score to API docs. Additional readability and minor docs bug fixes
 - PR #2468: Add `_n_features_in_` attribute to all single GPU estimators that implement fit
+- PR #2480: Moving MNMG glm and solvers to cuml
+- PR #2490: Moving MNMG KMeans to cuml
 - PR #2483: Moving MNMG KNN to cuml
 - PR #2492: Adding additional assertions to mnmg nearest neighbors pytests
 - PR #2439: Update dask RF code to have print_detailed function
 - PR #2431: Match output of classifier predict with target dtype
+- PR #2237: Refactor RF cython code
+- PR #2513: Fixing LGTM Analysis Issues
+- PR #2099: Raise an error when float64 data is used with dask RF
+- PR #2499: Provide access to `cuml.DBSCAN` core samples
+- PR #2526: Removing PCA TSQR as a solver due to scalability issues
+- PR #2536: Update conda upload versions for new supported CUDA/Python
+- PR #2538: Remove Protobuf dependency
+- PR #2553: Test pickle protocol 5 support
+- PR #2570: Accepting single df or array input in train_test_split
+- PR #2566: Remove deprecated cuDF from_gpu_matrix calls
+- PR #2583: findpackage.cmake.in template for cmake dependencies
+- PR #2577: Fully removing NVGraph dependency for CUDA 11 compatibility
+- PR #2575: Speed up TfidfTransformer
+- PR #2584: Removing dependency on sklearn's NotFittedError
+- PR #2591: Generate benchmark datsets using `cuml.datasets`
+- PR #2548: Fix limitation on number of rows usable with tSNE and refactor memory allocation
+- PR #2589: including cuda-11 build fixes into raft
+- PR #2487: Set classes_ attribute during classifier fit
+- PR #2605: Reduce memory usage in tSNE
 
 ## Bug Fixes
 - PR #2369: Update RF code to fix set_params memory leak
@@ -61,7 +92,25 @@
 - PR #2494: Set QN regularization strength consistent with scikit-learn
 - PR #2486: Fix cupy input to kmeans init
 - PR #2497: Changes to accomodate cuDF unsigned categorical changes
+- PR #2209: Fix FIL benchmark for gpuarray-c input
 - PR #2507: Import `treelite.sklearn`
+- PR #2532: Updating doxygen in new MG headers
+- PR #2521: Fixing invalid smem calculation in KNeighborsCLassifier
+- PR #2515: Increase tolerance for LogisticRegression test
+- PR #2545: Fix documentation of n_iter_without_progress in tSNE Python bindings
+- PR #2543: Improve numerical stability of QN solver
+- PR #2544: Fix Barnes-Hut tSNE not using specified post_learning_rate
+- PR #2558: Disabled a long-running FIL test
+- PR #2540: Update default value for n_epochs in UMAP to match documentation & sklearn API
+- PR #2535: Fix issue with incorrect docker image being used in local build script
+- PR #2542: Fix small memory leak in TSNE
+- PR #2552: Fixed the length argument of updateDevice calls in RF test
+- PR #2563: Update scipy call for arima gradient test
+- PR #2569: Fix for cuDF update
+- PR #2508: Use keyword parameters in sklearn.datasets.make_* functions
+- PR #2586: Fix SVC decision function data type
+- PR #2573: Considering managed memory as device type on checking for KMeans
+- PR #2574: Fixing include path in `tsvd_mg.pyx`
 
 # cuML 0.14.0 (03 Jun 2020)
 

ci/cpu/prebuild.sh

@@ -2,7 +2,7 @@
 
 export BUILD_CUML=1
 
-if [[ "$PYTHON" == "3.6" ]]; then
+if [[ "$PYTHON" == "3.7" ]]; then
     export BUILD_LIBCUML=1
 else
     export BUILD_LIBCUML=0

ci/gpu/build.sh

@@ -46,7 +46,7 @@ conda install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
       "cudatoolkit=${CUDA_REL}" \
       "cudf=${MINOR_VERSION}" \
       "rmm=${MINOR_VERSION}" \
-      "libcumlprims=0.15.0a200622" \
+      "libcumlprims=0.15.0a200720" \
       "dask-cudf=${MINOR_VERSION}" \
       "dask-cuda=${MINOR_VERSION}" \
       "ucx-py=${MINOR_VERSION}" \

ci/local/build.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 
-DOCKER_IMAGE="gpuci/rapidsai:cuda10.0-ubuntu16.04-gcc5-py3.6"
+GIT_DESCRIBE_TAG=`git describe --tags`
+MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
+
+DOCKER_IMAGE="gpuci/rapidsai:${MINOR_VERSION}-cuda10.1-devel-ubuntu16.04-py3.7"
 REPO_PATH=${PWD}
 RAPIDS_DIR_IN_CONTAINER="/rapids"
 CPP_BUILD_DIR="cuML/build"

ci/mg/build.sh

@@ -51,7 +51,6 @@ conda install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
       "lapack" \
       "cmake==3.14.3" \
       "umap-learn" \
-      "protobuf>=3.4.1,<4.0.0" \
       "nccl>=2.5" \
       "dask>=2.12.0" \
       "distributed>=2.12.0" \

conda/environments/cuml_dev_cuda10.0.yml

@@ -23,18 +23,19 @@ dependencies:
 - distributed>=2.12.0
 - dask-cuda=0.15*
 - dask-cudf=0.15*
+- dask-ml
 - ucx-py=0.15*
 - nccl>=2.5
-- libcumlprims=0.15.0a200622
+- libcumlprims=0.15.0a200720
 - statsmodels
-- protobuf >=3.4.1,<4.0.0
 - treelite=0.92
 - doxygen
 - sphinx
 - sphinx_rtd_theme
 - numpydoc
 - nbsphinx
 - recommonmark
+- ipython
 - pip
 - pip:
     - sphinx_markdown_tables

conda/environments/cuml_dev_cuda10.1.yml

@@ -23,18 +23,19 @@ dependencies:
 - distributed>=2.12.0
 - dask-cuda=0.15*
 - dask-cudf=0.15*
+- dask-ml
 - ucx-py=0.15*
 - nccl>=2.5
-- libcumlprims=0.15.0a200622
+- libcumlprims=0.15.0a200720
 - statsmodels
-- protobuf >=3.4.1,<4.0.0
 - treelite=0.92
 - doxygen
 - sphinx
 - sphinx_rtd_theme
 - numpydoc
 - nbsphinx
 - recommonmark
+- ipython
 - pip
 - pip:
     - sphinx_markdown_tables

conda/environments/cuml_dev_cuda10.2.yml

@@ -23,18 +23,19 @@ dependencies:
 - distributed>=2.12.0
 - dask-cuda=0.15*
 - dask-cudf=0.15*
+- dask-ml
 - ucx-py=0.15*
 - nccl>=2.5
-- libcumlprims=0.15.0a200622
+- libcumlprims=0.15.0a200720
 - statsmodels
-- protobuf >=3.4.1,<4.0.0
 - treelite=0.92
 - doxygen
 - sphinx
 - sphinx_rtd_theme
 - numpydoc
 - nbsphinx
 - recommonmark
+- ipython
 - pip
 - pip:
     - sphinx_markdown_tables

conda/recipes/cuml/meta.yaml

@@ -28,19 +28,18 @@ requirements:
     - setuptools
     - cython>=0.29,<0.30
     - cmake>=3.14
-    - protobuf >=3.4.1,<4.0.0
     - treelite=0.92
     - cudf {{ minor_version }}
     - libcuml={{ version }}
-    - libcumlprims 0.15.0a200622
+    - libcumlprims 0.15.0a200720
     - cudatoolkit {{ cuda_version }}.*
     - ucx-py {{ minor_version }}
   run:
     - python x.x
     - cudf {{ minor_version }}
     - dask-cudf {{ minor_version }}
     - libcuml={{ version }}
-    - libcumlprims=0.15.0a200622
+    - libcumlprims=0.15.0a200720
     - treelite=0.92
     - cupy>=7,<8.0.0a0
     - nccl>=2.5

conda/recipes/libcuml/meta.yaml

@@ -34,12 +34,11 @@ requirements:
     - cudf {{ minor_version }}
     - cudatoolkit {{ cuda_version }}.*
     - ucx-py {{ minor_version }}
-    - libcumlprims=0.15.0a200622
+    - libcumlprims=0.15.0a200720
     - lapack
-    - protobuf >=3.4.1,<4.0.0
     - treelite=0.92
   run:
-    - libcumlprims=0.15.0a200622
+    - libcumlprims=0.15.0a200720
     - cudf {{ minor_version }}
     - nccl>=2.5
     - ucx-py {{ minor_version }}

cpp/CMakeLists.txt

@@ -164,6 +164,32 @@ endif(BUILD_CUML_MG_TESTS AND NOT SINGLEGPU)
 ##############################################################################
 # - Requirements -------------------------------------------------------------
 
+# Create FindPackage.cmake files to use find(package) functionality for
+# dependencies
+
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/cmake/templates)
+
+GENERATE_FIND_MODULE(NAME cumlprims_mg
+                     HEADER_NAME cumlprims.hpp
+                     LIBRARY_NAME cumlprims
+                     LOCATION cumlprims)
+
+GENERATE_FIND_MODULE(NAME NCCL
+                     HEADER_NAME nccl.h
+                     LIBRARY_NAME nccl)
+
+GENERATE_FIND_MODULE(NAME RMM
+                     HEADER_NAME rmm/device_buffer.hpp
+                     LIBRARY_NAME rmm)
+
+GENERATE_FIND_MODULE(NAME UCX
+                     HEADER_NAME ucp/api/ucp.h
+                     LIBRARY_NAME ucp)
+
+set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_BINARY_DIR}/cmake)
+
+# append additional directories to CMAKE_MODULE_PATH if we need other modules here
+
 # todo: change for FindCUDAToolkit when CMake 3.17 is the minimum required
 # https://github.com/rapidsai/cuml/issues/2194
 find_package(CUDA 10.0 REQUIRED)
@@ -181,17 +207,10 @@ else()
   message(STATUS "Manually setting BLAS to ${BLAS_LIBRARIES}")
 endif()
 
-set(Protobuf_USE_STATIC_LIBS ON)
-find_package(Protobuf REQUIRED)
-
-set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
 ##############################################################################
 # - External Dependencies-----------------------------------------------------
 
-set(CUDA_nvgraph_LIBRARY ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libnvgraph.so CACHE STRING
-  "Path to nvGraph lib")
-
 set(CUML_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include CACHE STRING
   "Path to cuml include directories")
 
@@ -304,6 +323,7 @@ set(CUML_INCLUDE_DIRECTORIES
   ${CUB_DIR}/src/cub
   ${SPDLOG_DIR}/src/spdlog/include
   ${RAFT_DIR}/cpp/include
+  ${RMM_INCLUDE_DIRS}
   )
 
 set(CUML_PUBLIC_LINK_LIBRARIES
@@ -312,22 +332,21 @@ set(CUML_PUBLIC_LINK_LIBRARIES
   ${CUDA_cusolver_LIBRARY}
   ${CUDA_CUDART_LIBRARY}
   ${CUDA_cusparse_LIBRARY}
-  ${CUDA_nvgraph_LIBRARY}
   )
 
 set(CUML_PRIVATE_LINK_LIBRARIES
-  ${Protobuf_LIBRARIES}
   faisslib
   treelite::treelite
   treelite::treelite_runtime
+  RMM::RMM
   )
 
 if(ENABLE_CUMLPRIMS_MG)
   list(APPEND CUML_INCLUDE_DIRECTORIES
-       ${CUMLPRIMS_MG_INCLUDE_DIRS})
+       ${cumlprims_mg_INCLUDE_DIRS})
 
   list(APPEND CUML_PRIVATE_LINK_LIBRARIES
-       CUMLPRIMS_MG::CUMLPRIMS_MG)
+       cumlprims_mg::cumlprims_mg)
 
 endif(ENABLE_CUMLPRIMS_MG)
 
@@ -379,9 +398,17 @@ if(BUILD_CUML_CPP_LIBRARY)
   if(NOT SINGLEGPU)
     target_sources(${CUML_CPP_TARGET}
       PRIVATE
+        src/glm/ols_mg.cu
+        src/glm/preprocess_mg.cu
+        src/glm/ridge_mg.cu
+        src/kmeans/kmeans_mg.cu
         src/knn/knn_mg.cu
+        src/knn/knn_opg_common.cu
+        src/knn/knn_classify_mg.cu
+        src/knn/knn_regress_mg.cu
         src/pca/pca_mg.cu
         src/pca/sign_flip_mg.cu
+        src/solver/cd_mg.cu
         src/tsvd/tsvd_mg.cu
     )
   endif(NOT SINGLEGPU)