From 868917056f5959a3d44e07bc69dae8cb4e0170bc Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sat, 11 Apr 2020 07:42:56 +0800
Subject: [PATCH 01/20] dask cudf inplace prediction.

---
 python-package/xgboost/dask.py         |  2 --
 tests/ci_build/Dockerfile.cudf         |  2 +-
 tests/ci_build/Dockerfile.gpu          |  2 +-
 tests/python-gpu/test_gpu_with_dask.py | 10 ++++------
 4 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/python-package/xgboost/dask.py b/python-package/xgboost/dask.py
index fa20605c3a18..bd30223545d2 100644
--- a/python-package/xgboost/dask.py
+++ b/python-package/xgboost/dask.py
@@ -623,8 +623,6 @@ def mapped_predict(data, is_df):
         if is_df:
             if lazy_isinstance(data, 'cudf.core.dataframe', 'DataFrame'):
                 import cudf     # pylint: disable=import-error
-                # There's an error with cudf saying `concat_cudf` got an
-                # expected argument `ignore_index`. So this is not yet working.
                 prediction = cudf.DataFrame({'prediction': prediction},
                                             dtype=numpy.float32)
             else:
diff --git a/tests/ci_build/Dockerfile.cudf b/tests/ci_build/Dockerfile.cudf
index 72039968da12..638e6d457fe0 100644
--- a/tests/ci_build/Dockerfile.cudf
+++ b/tests/ci_build/Dockerfile.cudf
@@ -18,7 +18,7 @@ ENV PATH=/opt/python/bin:$PATH
 # Create new Conda environment with cuDF and dask
 RUN \
     conda create -n cudf_test -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \
-        cudf=0.9 python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask dask-cuda cupy
+           python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask dask-cuda cupy cudf
 
 # Install other Python packages
 RUN \
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index dc92906e9166..d30f500d75df 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -18,7 +18,7 @@ ENV PATH=/opt/python/bin:$PATH
 RUN \
     pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz && \
     pip install "dask[complete]" && \
-    conda install -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda dask-cuda
+    conda install -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda dask-cuda cudf dask-cudf
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/python-gpu/test_gpu_with_dask.py b/tests/python-gpu/test_gpu_with_dask.py
index 2de976ce0af3..e2e350744e14 100644
--- a/tests/python-gpu/test_gpu_with_dask.py
+++ b/tests/python-gpu/test_gpu_with_dask.py
@@ -51,16 +51,14 @@ def test_dask_dataframe(self):
                 predictions = dxgb.predict(client, out, dtrain).compute()
                 assert isinstance(predictions, np.ndarray)
 
-                # There's an error with cudf saying `concat_cudf` got an
-                # expected argument `ignore_index`.  So the test here is just
-                # place holder.
-
-                # series_predictions = dxgb.inplace_predict(client, out, X)
-                # assert isinstance(series_predictions, dd.Series)
+                series_predictions = dxgb.inplace_predict(client, out, X)
+                assert isinstance(series_predictions, dd.Series)
+                series_predictions = series_predictions.compute()
 
                 single_node = out['booster'].predict(
                     xgboost.DMatrix(X.compute()))
                 cupy.testing.assert_allclose(single_node, predictions)
+                cupy.testing.assert_allclose(single_node, series_predictions)
 
     @pytest.mark.skipif(**tm.no_cupy())
     def test_dask_array(self):

From 12b31c502de7e71505ee7186446f3040424a3541 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Sun, 12 Apr 2020 19:03:55 +0800
Subject: [PATCH 02/20] Change CI script.

---
 tests/ci_build/Dockerfile.cudf | 6 ++++--
 tests/ci_build/Dockerfile.gpu  | 2 +-
 tests/ci_build/test_python.sh  | 3 ++-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/ci_build/Dockerfile.cudf b/tests/ci_build/Dockerfile.cudf
index 638e6d457fe0..e0a5b5e1d1e2 100644
--- a/tests/ci_build/Dockerfile.cudf
+++ b/tests/ci_build/Dockerfile.cudf
@@ -17,8 +17,10 @@ ENV PATH=/opt/python/bin:$PATH
 
 # Create new Conda environment with cuDF and dask
 RUN \
-    conda create -n cudf_test -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \
-           python=3.7 anaconda::cudatoolkit=$CUDA_VERSION dask dask-cuda cupy cudf
+    conda create -n cudf_test python=3.7 && \
+    source activate cudf_test && \
+    conda install -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \
+          anaconda::cudatoolkit=$CUDA_VERSION dask dask-cuda dask-cudf cupy cudf
 
 # Install other Python packages
 RUN \
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index d30f500d75df..f896fd69bc5f 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -18,7 +18,7 @@ ENV PATH=/opt/python/bin:$PATH
 RUN \
     pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz && \
     pip install "dask[complete]" && \
-    conda install -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda dask-cuda cudf dask-cudf
+    conda install -c rapidsai -c conda-forge -c anaconda dask-cuda
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index b8b723c5b97d..b5efda2dfd73 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -44,7 +44,8 @@ case "$suite" in
   cudf)
     source activate cudf_test
     install_xgboost
-    pytest -v -s --fulltrace -m "not mgpu" tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py
+    pytest -v -s --fulltrace -m "not mgpu" tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py \
+	   tests/python-gpu/test_gpu_with_dask.py::TestDistributedGPU::test_dask_dataframe
     ;;
 
   cpu)

From bd50317c1e8e90d502b82a157b892dc928f5dd16 Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Mon, 13 Apr 2020 22:15:59 -0700
Subject: [PATCH 03/20] Remove Dockerfile.release, since it's not used anywhere

---
 tests/ci_build/Dockerfile.release | 31 -------------------------------
 1 file changed, 31 deletions(-)
 delete mode 100644 tests/ci_build/Dockerfile.release

diff --git a/tests/ci_build/Dockerfile.release b/tests/ci_build/Dockerfile.release
deleted file mode 100644
index f1da067898ca..000000000000
--- a/tests/ci_build/Dockerfile.release
+++ /dev/null
@@ -1,31 +0,0 @@
-FROM centos:6
-
-# Install all basic requirements
-RUN \
-    yum -y update && \
-    yum install -y graphviz tar unzip wget xz git && \
-    # Python
-    wget https://repo.continuum.io/miniconda/Miniconda2-4.3.27-Linux-x86_64.sh && \
-    bash Miniconda2-4.3.27-Linux-x86_64.sh -b -p /opt/python
-
-ENV PATH=/opt/python/bin:$PATH
-
-# Install Python packages
-RUN \
-    conda install numpy scipy pandas matplotlib pytest scikit-learn && \
-    pip install pytest wheel auditwheel graphviz
-
-ENV GOSU_VERSION 1.10
-
-# Install lightweight sudo (not bound to TTY)
-RUN set -ex; \
-    wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
-    chmod +x /usr/local/bin/gosu && \
-    gosu nobody true
-
-# Default entry-point to use if running locally
-# It will preserve attributes of created files
-COPY entrypoint.sh /scripts/
-
-WORKDIR /workspace
-ENTRYPOINT ["/scripts/entrypoint.sh"]

From e94fe522b3b510675f48f9c63b12f3843938a1e8 Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Mon, 13 Apr 2020 22:45:25 -0700
Subject: [PATCH 04/20] Use latest Miniconda version

---
 tests/ci_build/Dockerfile.cpu       | 4 ++--
 tests/ci_build/Dockerfile.cudf      | 4 ++--
 tests/ci_build/Dockerfile.gpu       | 4 ++--
 tests/ci_build/Dockerfile.gpu_build | 4 ++--
 tests/ci_build/Dockerfile.jvm       | 4 ++--
 tests/ci_build/Dockerfile.jvm_cross | 4 ++--
 6 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
index 02b6024bc25e..e804d1b4896b 100644
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@@ -13,8 +13,8 @@ RUN \
     wget -nv -nc https://cmake.org/files/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.0-Linux-x86_64.sh --no-check-certificate && \
     bash cmake-$CMAKE_VERSION.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+    wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3.sh -b -p /opt/python
 
 ENV PATH=/opt/python/bin:$PATH
 
diff --git a/tests/ci_build/Dockerfile.cudf b/tests/ci_build/Dockerfile.cudf
index e0a5b5e1d1e2..9bc56ac03ad3 100644
--- a/tests/ci_build/Dockerfile.cudf
+++ b/tests/ci_build/Dockerfile.cudf
@@ -10,8 +10,8 @@ RUN \
     apt-get update && \
     apt-get install -y wget unzip bzip2 libgomp1 build-essential && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+    wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3.sh -b -p /opt/python
 
 ENV PATH=/opt/python/bin:$PATH
 
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index f896fd69bc5f..e58df753e6da 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -9,8 +9,8 @@ RUN \
     apt-get update && \
     apt-get install -y wget unzip bzip2 libgomp1 build-essential && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python
+    wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3.sh -b -p /opt/python
 
 ENV PATH=/opt/python/bin:$PATH
 
diff --git a/tests/ci_build/Dockerfile.gpu_build b/tests/ci_build/Dockerfile.gpu_build
index 052408f616cf..7e7402b25763 100644
--- a/tests/ci_build/Dockerfile.gpu_build
+++ b/tests/ci_build/Dockerfile.gpu_build
@@ -12,8 +12,8 @@ RUN \
     yum -y update && \
     yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3.sh -b -p /opt/python && \
     # CMake
     wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
     bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr
diff --git a/tests/ci_build/Dockerfile.jvm b/tests/ci_build/Dockerfile.jvm
index 6bbcbc800a4a..33295ab5cbac 100644
--- a/tests/ci_build/Dockerfile.jvm
+++ b/tests/ci_build/Dockerfile.jvm
@@ -8,8 +8,8 @@ RUN \
     yum -y update && \
     yum install -y devtoolset-4-gcc devtoolset-4-binutils devtoolset-4-gcc-c++ && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3.sh -b -p /opt/python && \
     # CMake
     wget -nv -nc https://cmake.org/files/v3.12/cmake-3.12.0-Linux-x86_64.sh --no-check-certificate && \
     bash cmake-3.12.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross
index e5b4479a84d0..7cb953bb0faa 100644
--- a/tests/ci_build/Dockerfile.jvm_cross
+++ b/tests/ci_build/Dockerfile.jvm_cross
@@ -13,8 +13,8 @@ RUN \
     apt-get update && \
     apt-get install -y tar unzip wget openjdk-$JDK_VERSION-jdk libgomp1 && \
     # Python
-    wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \
+    wget -O Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash Miniconda3.sh -b -p /opt/python && \
     /opt/python/bin/pip install awscli && \
     # Maven
     wget https://archive.apache.org/dist/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \

From e044d8f8c3d43fabdc236e9dcd783e26d59ae68c Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Mon, 13 Apr 2020 23:37:55 -0700
Subject: [PATCH 05/20] Use Conda exclusively in CUDF and GPU containers

---
 tests/ci_build/Dockerfile.cudf | 14 ++++----------
 tests/ci_build/Dockerfile.gpu  |  6 +++---
 tests/ci_build/test_python.sh  |  2 ++
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/tests/ci_build/Dockerfile.cudf b/tests/ci_build/Dockerfile.cudf
index 9bc56ac03ad3..b5178632be9c 100644
--- a/tests/ci_build/Dockerfile.cudf
+++ b/tests/ci_build/Dockerfile.cudf
@@ -15,17 +15,11 @@ RUN \
 
 ENV PATH=/opt/python/bin:$PATH
 
-# Create new Conda environment with cuDF and dask
+# Create new Conda environment with cuDF, Dask, and cuPy
 RUN \
-    conda create -n cudf_test python=3.7 && \
-    source activate cudf_test && \
-    conda install -c rapidsai -c nvidia -c numba -c conda-forge -c anaconda \
-          anaconda::cudatoolkit=$CUDA_VERSION dask dask-cuda dask-cudf cupy cudf
-
-# Install other Python packages
-RUN \
-    source activate cudf_test && \
-    pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
+    conda create -n cudf_test -c rapidsai -c nvidia -c conda-forge -c defaults \
+        python=3.7 cudf cudatoolkit=$CUDA_VERSION dask dask-cuda dask-cudf cupy \
+        numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index e58df753e6da..0c9e1ab23a13 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -16,9 +16,9 @@ ENV PATH=/opt/python/bin:$PATH
 
 # Install Python packages
 RUN \
-    pip install numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz && \
-    pip install "dask[complete]" && \
-    conda install -c rapidsai -c conda-forge -c anaconda dask-cuda
+    conda create -n gpu_test -c rapidsai -c nvidia -c conda-forge -c defaults \
+        python=3.7 cudatoolkit=$CUDA_VERSION dask dask-cuda numpy pytest scipy \
+        scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index b5efda2dfd73..bd1962b9982a 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -28,11 +28,13 @@ function install_xgboost {
 # Run specified test suite
 case "$suite" in
   gpu)
+    source activate gpu_test
     install_xgboost
     pytest -v -s --fulltrace -m "not mgpu" tests/python-gpu
     ;;
 
   mgpu)
+    source activate gpu_test
     install_xgboost
     pytest -v -s --fulltrace -m "mgpu" tests/python-gpu
     cd tests/distributed

From 73f073ba83672af152be1f1bfd152bd180e182e1 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 14 Apr 2020 16:30:51 +0800
Subject: [PATCH 06/20] Correct pytest mark.

---
 tests/python-gpu/test_gpu_with_dask.py         | 2 ++
 tests/python-gpu/test_monotonic_constraints.py | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/python-gpu/test_gpu_with_dask.py b/tests/python-gpu/test_gpu_with_dask.py
index e2e350744e14..826875b2c00b 100644
--- a/tests/python-gpu/test_gpu_with_dask.py
+++ b/tests/python-gpu/test_gpu_with_dask.py
@@ -27,6 +27,7 @@ class TestDistributedGPU(unittest.TestCase):
     @pytest.mark.skipif(**tm.no_cudf())
     @pytest.mark.skipif(**tm.no_dask_cudf())
     @pytest.mark.skipif(**tm.no_dask_cuda())
+    @pytest.mark.mgpu
     def test_dask_dataframe(self):
         with LocalCUDACluster() as cluster:
             with Client(cluster) as client:
@@ -61,6 +62,7 @@ def test_dask_dataframe(self):
                 cupy.testing.assert_allclose(single_node, series_predictions)
 
     @pytest.mark.skipif(**tm.no_cupy())
+    @pytest.mark.mgpu
     def test_dask_array(self):
         with LocalCUDACluster() as cluster:
             with Client(cluster) as client:
diff --git a/tests/python-gpu/test_monotonic_constraints.py b/tests/python-gpu/test_monotonic_constraints.py
index 69063f11bacf..6208e137851a 100644
--- a/tests/python-gpu/test_monotonic_constraints.py
+++ b/tests/python-gpu/test_monotonic_constraints.py
@@ -35,7 +35,6 @@ def assert_constraint(constraint, tree_method):
         assert non_increasing(pred)
 
 
-@pytest.mark.gpu
 class TestMonotonicConstraints(unittest.TestCase):
     def test_exact(self):
         assert_constraint(1, 'exact')

From 76cbef7e66e389d899671cee9b856eec5e96dfcb Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 14 Apr 2020 16:34:36 +0800
Subject: [PATCH 07/20] Use `python-kubernetes`.

---
 tests/ci_build/Dockerfile.cpu  | 2 +-
 tests/ci_build/Dockerfile.cudf | 2 +-
 tests/ci_build/Dockerfile.gpu  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
index ad4464e3f4a4..9c787eb9360f 100644
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@@ -29,7 +29,7 @@ RUN conda create -n py35 python=3.5 && \
 RUN \
     pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh \
     		recommonmark guzzle_sphinx_theme mock breathe graphviz \
-		pytest scikit-learn wheel kubernetes urllib3 jsonschema boto3 && \
+		pytest scikit-learn wheel python-kubernetes urllib3 jsonschema boto3 && \
     pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl && \
     pip install "dask[complete]"
 
diff --git a/tests/ci_build/Dockerfile.cudf b/tests/ci_build/Dockerfile.cudf
index b5178632be9c..7661cebe047d 100644
--- a/tests/ci_build/Dockerfile.cudf
+++ b/tests/ci_build/Dockerfile.cudf
@@ -19,7 +19,7 @@ ENV PATH=/opt/python/bin:$PATH
 RUN \
     conda create -n cudf_test -c rapidsai -c nvidia -c conda-forge -c defaults \
         python=3.7 cudf cudatoolkit=$CUDA_VERSION dask dask-cuda dask-cudf cupy \
-        numpy pytest scipy scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
+        numpy pytest scipy scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index 0c9e1ab23a13..4f02c4ee2d06 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -18,7 +18,7 @@ ENV PATH=/opt/python/bin:$PATH
 RUN \
     conda create -n gpu_test -c rapidsai -c nvidia -c conda-forge -c defaults \
         python=3.7 cudatoolkit=$CUDA_VERSION dask dask-cuda numpy pytest scipy \
-        scikit-learn pandas matplotlib wheel kubernetes urllib3 graphviz
+        scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 

From 9e20bb79e7a03bbe902e6ef8ff5122a76eaa978d Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Tue, 14 Apr 2020 02:11:43 -0700
Subject: [PATCH 08/20] Use kubernetes with pip

---
 tests/ci_build/Dockerfile.cpu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
index 9c787eb9360f..ad4464e3f4a4 100644
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@@ -29,7 +29,7 @@ RUN conda create -n py35 python=3.5 && \
 RUN \
     pip install pyyaml cpplint pylint astroid sphinx numpy scipy pandas matplotlib sh \
     		recommonmark guzzle_sphinx_theme mock breathe graphviz \
-		pytest scikit-learn wheel python-kubernetes urllib3 jsonschema boto3 && \
+		pytest scikit-learn wheel kubernetes urllib3 jsonschema boto3 && \
     pip install https://h2o-release.s3.amazonaws.com/datatable/stable/datatable-0.7.0/datatable-0.7.0-cp37-cp37m-linux_x86_64.whl && \
     pip install "dask[complete]"
 

From 269c9f125939ad667f077a9660f6dde49ff7af66 Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Tue, 14 Apr 2020 02:12:31 -0700
Subject: [PATCH 09/20] Do not install 'cudatoolkit' conda package for GPU
 container, as cuDF is not used there

---
 tests/ci_build/Dockerfile.gpu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index 4f02c4ee2d06..4ec61ab4d90a 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -17,8 +17,8 @@ ENV PATH=/opt/python/bin:$PATH
 # Install Python packages
 RUN \
     conda create -n gpu_test -c rapidsai -c nvidia -c conda-forge -c defaults \
-        python=3.7 cudatoolkit=$CUDA_VERSION dask dask-cuda numpy pytest scipy \
-        scikit-learn pandas matplotlib wheel python-kubernetes urllib3 graphviz
+        python=3.7 dask dask-cuda numpy pytest scipy  scikit-learn pandas \
+        matplotlib wheel python-kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 

From 4288462c7637dd119f3c12310d22856a4df9dff3 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 14 Apr 2020 20:14:11 +0800
Subject: [PATCH 10/20] Strict device id checks.

---
 python-package/xgboost/core.py         | 5 ++++-
 python-package/xgboost/dask.py         | 5 +++++
 tests/ci_build/Dockerfile.gpu          | 4 ++--
 tests/ci_build/test_python.sh          | 8 ++++----
 tests/python-gpu/test_gpu_with_dask.py | 7 ++++++-
 5 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/python-package/xgboost/core.py b/python-package/xgboost/core.py
index 688904ce1f34..dab38a26d547 100644
--- a/python-package/xgboost/core.py
+++ b/python-package/xgboost/core.py
@@ -210,9 +210,12 @@ def ctypes2numpy(cptr, length, dtype):
 def ctypes2cupy(cptr, length, dtype):
     """Convert a ctypes pointer array to a cupy array."""
     import cupy                 # pylint: disable=import-error
-    mem = cupy.zeros(length.value, dtype=dtype, order='C')
     addr = ctypes.cast(cptr, ctypes.c_void_p).value
     # pylint: disable=c-extension-no-member,no-member
+    device = cupy.cuda.runtime.pointerGetAttributes(addr).device
+    mem = cupy.zeros(length.value, dtype=dtype, order='C')
+    assert mem.device.id == device
+    assert mem.flags.c_contiguous
     cupy.cuda.runtime.memcpy(
         mem.__cuda_array_interface__['data'][0], addr,
         length.value * ctypes.sizeof(ctypes.c_float),
diff --git a/python-package/xgboost/dask.py b/python-package/xgboost/dask.py
index bd30223545d2..3316a3cf6b23 100644
--- a/python-package/xgboost/dask.py
+++ b/python-package/xgboost/dask.py
@@ -101,6 +101,11 @@ def concat(value):              # pylint: disable=too-many-return-statements
         return CUDF_concat(value, axis=0)
     if lazy_isinstance(value[0], 'cupy.core.core', 'ndarray'):
         import cupy             # pylint: disable=import-error
+        # pylint: disable=c-extension-no-member,no-member
+        d = cupy.cuda.runtime.getDevice()
+        for v in value:
+            d_v = v.device.id
+            assert d_v == d, 'Concatenating arrays on different devices.'
         return cupy.concatenate(value, axis=0)
     return dd.multi.concat(list(value), axis=0)
 
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index 4ec61ab4d90a..70169d724369 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -17,8 +17,8 @@ ENV PATH=/opt/python/bin:$PATH
 # Install Python packages
 RUN \
     conda create -n gpu_test -c rapidsai -c nvidia -c conda-forge -c defaults \
-        python=3.7 dask dask-cuda numpy pytest scipy  scikit-learn pandas \
-        matplotlib wheel python-kubernetes urllib3 graphviz
+        python=3.7 dask dask-cuda numpy pytest scipy scikit-learn pandas \
+        matplotlib wheel python-kubernetes urllib3 graphviz dask-cudf cudf cupy
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index bd1962b9982a..da41a090c399 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -30,23 +30,23 @@ case "$suite" in
   gpu)
     source activate gpu_test
     install_xgboost
-    pytest -v -s --fulltrace -m "not mgpu" tests/python-gpu
+    pytest -v -s -rxXs --fulltrace -m "not mgpu" tests/python-gpu
     ;;
 
   mgpu)
     source activate gpu_test
     install_xgboost
-    pytest -v -s --fulltrace -m "mgpu" tests/python-gpu
+    pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu
     cd tests/distributed
     ./runtests-gpu.sh
     cd -
-    pytest -v -s --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py
     ;;
 
   cudf)
     source activate cudf_test
     install_xgboost
-    pytest -v -s --fulltrace -m "not mgpu" tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py \
+    pytest -v -s -rxXs --fulltrace -m "not mgpu" \
+           tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py \
 	   tests/python-gpu/test_gpu_with_dask.py::TestDistributedGPU::test_dask_dataframe
     ;;
 
diff --git a/tests/python-gpu/test_gpu_with_dask.py b/tests/python-gpu/test_gpu_with_dask.py
index 826875b2c00b..7cc45a428459 100644
--- a/tests/python-gpu/test_gpu_with_dask.py
+++ b/tests/python-gpu/test_gpu_with_dask.py
@@ -58,6 +58,7 @@ def test_dask_dataframe(self):
 
                 single_node = out['booster'].predict(
                     xgboost.DMatrix(X.compute()))
+
                 cupy.testing.assert_allclose(single_node, predictions)
                 cupy.testing.assert_allclose(single_node, series_predictions)
 
@@ -82,8 +83,12 @@ def test_dask_array(self):
                 single_node = out['booster'].predict(
                     xgboost.DMatrix(X.compute()))
                 np.testing.assert_allclose(single_node, from_dmatrix)
+                device = cupy.cuda.runtime.getDevice()
+                assert device == inplace_predictions.device.id
+                single_node = cupy.array(single_node)
+                assert device == single_node.device.id
                 cupy.testing.assert_allclose(
-                    cupy.array(single_node),
+                    single_node,
                     inplace_predictions)
 
 

From e1ebde0646be09c868411fe816888af8437f3f35 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 14 Apr 2020 22:29:29 +0800
Subject: [PATCH 11/20] Fix latest cudf.

---
 tests/python/test_predict.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/python/test_predict.py b/tests/python/test_predict.py
index d3258fdc5685..088cf2912f5c 100644
--- a/tests/python/test_predict.py
+++ b/tests/python/test_predict.py
@@ -12,10 +12,10 @@ def run_threaded_predict(X, rows, predict_func):
     per_thread = 20
     with ThreadPoolExecutor(max_workers=10) as e:
         for i in range(0, rows, int(rows / per_thread)):
-            try:
+            if hasattr(X, 'iloc'):
+                predictor = X.iloc[i:i+per_thread, :]
+            else:
                 predictor = X[i:i+per_thread, ...]
-            except TypeError:
-                predictor = X.iloc[i:i+per_thread, ...]
             f = e.submit(predict_func, predictor)
             results.append(f)
 

From c652cc038cc5c6db5e66f9671c582610b8f6238e Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 14 Apr 2020 22:39:40 +0800
Subject: [PATCH 12/20] Add skip marks.

---
 tests/python-gpu/test_gpu_prediction.py        | 2 ++
 tests/python-gpu/test_monotonic_constraints.py | 9 ++++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tests/python-gpu/test_gpu_prediction.py b/tests/python-gpu/test_gpu_prediction.py
index 211cb115ddb9..937c624df91e 100644
--- a/tests/python-gpu/test_gpu_prediction.py
+++ b/tests/python-gpu/test_gpu_prediction.py
@@ -62,6 +62,7 @@ def non_decreasing(self, L):
 
     # Test case for a bug where multiple batch predictions made on a
     # test set produce incorrect results
+    @pytest.mark.skipif(**tm.no_sklearn())
     def test_multi_predict(self):
         from sklearn.datasets import make_regression
         from sklearn.model_selection import train_test_split
@@ -89,6 +90,7 @@ def test_multi_predict(self):
         assert np.allclose(predict0, predict1)
         assert np.allclose(predict0, cpu_predict)
 
+    @pytest.mark.skipif(**tm.no_sklearn())
     def test_sklearn(self):
         m, n = 15000, 14
         tr_size = 2500
diff --git a/tests/python-gpu/test_monotonic_constraints.py b/tests/python-gpu/test_monotonic_constraints.py
index 6208e137851a..9b44b951e6fd 100644
--- a/tests/python-gpu/test_monotonic_constraints.py
+++ b/tests/python-gpu/test_monotonic_constraints.py
@@ -1,12 +1,12 @@
-from __future__ import print_function
-
+import sys
 import numpy as np
-from sklearn.datasets import make_regression
 
 import unittest
 import pytest
 
 import xgboost as xgb
+sys.path.append("tests/python")
+import testing as tm
 
 rng = np.random.RandomState(1994)
 
@@ -20,6 +20,7 @@ def non_increasing(L):
 
 
 def assert_constraint(constraint, tree_method):
+    from sklearn.datasets import make_regression
     n = 1000
     X, y = make_regression(n, random_state=rng, n_features=1, n_informative=1)
     dtrain = xgb.DMatrix(X, y)
@@ -36,10 +37,12 @@ def assert_constraint(constraint, tree_method):
 
 
 class TestMonotonicConstraints(unittest.TestCase):
+    @pytest.mark.skipif(**tm.no_sklearn())
     def test_exact(self):
         assert_constraint(1, 'exact')
         assert_constraint(-1, 'exact')
 
+    @pytest.mark.skipif(**tm.no_sklearn())
     def test_gpu_hist(self):
         assert_constraint(1, 'gpu_hist')
         assert_constraint(-1, 'gpu_hist')

From f8ed0c415c5fd369906b7a4939ffb3a4cc8be269 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Tue, 14 Apr 2020 23:30:18 +0800
Subject: [PATCH 13/20] Don't use raw memcpy.

---
 python-package/xgboost/core.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/python-package/xgboost/core.py b/python-package/xgboost/core.py
index dab38a26d547..72180cfd09d3 100644
--- a/python-package/xgboost/core.py
+++ b/python-package/xgboost/core.py
@@ -209,18 +209,22 @@ def ctypes2numpy(cptr, length, dtype):
 
 def ctypes2cupy(cptr, length, dtype):
     """Convert a ctypes pointer array to a cupy array."""
-    import cupy                 # pylint: disable=import-error
+    # pylint: disable=import-error
+    import cupy
+    from cupy.cuda.memory import MemoryPointer
+    from cupy.cuda.memory import UnownedMemory
     addr = ctypes.cast(cptr, ctypes.c_void_p).value
     # pylint: disable=c-extension-no-member,no-member
     device = cupy.cuda.runtime.pointerGetAttributes(addr).device
-    mem = cupy.zeros(length.value, dtype=dtype, order='C')
+    # The owner field is just used to keep the memory alive with ref count.  As
+    # unowned's life time is scoped within this function we don't need that.
+    unownd = UnownedMemory(
+        addr, length.value * ctypes.sizeof(ctypes.c_float), owner=None)
+    memptr = MemoryPointer(unownd, 0)
+    mem = cupy.ndarray((length.value, ), dtype=np.float32, memptr=memptr)
     assert mem.device.id == device
-    assert mem.flags.c_contiguous
-    cupy.cuda.runtime.memcpy(
-        mem.__cuda_array_interface__['data'][0], addr,
-        length.value * ctypes.sizeof(ctypes.c_float),
-        cupy.cuda.runtime.memcpyDeviceToDevice)
-    return mem
+    arr = cupy.array(mem, copy=True)
+    return arr
 
 
 def ctypes2buffer(cptr, length):

From 30f792ea45da24eee5dc1ec0012d1235ddf0b166 Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Wed, 15 Apr 2020 03:21:53 +0800
Subject: [PATCH 14/20] Revert installation.

---
 tests/ci_build/Dockerfile.gpu | 4 ++--
 tests/ci_build/test_python.sh | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
index 70169d724369..4ec61ab4d90a 100644
--- a/tests/ci_build/Dockerfile.gpu
+++ b/tests/ci_build/Dockerfile.gpu
@@ -17,8 +17,8 @@ ENV PATH=/opt/python/bin:$PATH
 # Install Python packages
 RUN \
     conda create -n gpu_test -c rapidsai -c nvidia -c conda-forge -c defaults \
-        python=3.7 dask dask-cuda numpy pytest scipy scikit-learn pandas \
-        matplotlib wheel python-kubernetes urllib3 graphviz dask-cudf cudf cupy
+        python=3.7 dask dask-cuda numpy pytest scipy  scikit-learn pandas \
+        matplotlib wheel python-kubernetes urllib3 graphviz
 
 ENV GOSU_VERSION 1.10
 
diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index da41a090c399..3c714e355166 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -47,7 +47,8 @@ case "$suite" in
     install_xgboost
     pytest -v -s -rxXs --fulltrace -m "not mgpu" \
            tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py \
-	   tests/python-gpu/test_gpu_with_dask.py::TestDistributedGPU::test_dask_dataframe
+	   tests/python-gpu/test_gpu_predict.py::TestGPUPredict::test_inplace_predict_cupy \
+	   tests/python-gpu/test_gpu_predict.py::TestGPUPredict::test_inplace_predict_cudf
     ;;
 
   cpu)

From d503aac86fc372f890acf7c62caf574e173ebf9a Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Wed, 15 Apr 2020 03:30:34 +0800
Subject: [PATCH 15/20] type mapping.

---
 python-package/xgboost/core.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/python-package/xgboost/core.py b/python-package/xgboost/core.py
index 72180cfd09d3..057b32365cf1 100644
--- a/python-package/xgboost/core.py
+++ b/python-package/xgboost/core.py
@@ -213,15 +213,25 @@ def ctypes2cupy(cptr, length, dtype):
     import cupy
     from cupy.cuda.memory import MemoryPointer
     from cupy.cuda.memory import UnownedMemory
+    CUPY_TO_CTYPES_MAPPING = {
+        cupy.float32: ctypes.c_float,
+        cupy.uint32: ctypes.c_uint
+    }
+    if dtype not in CUPY_TO_CTYPES_MAPPING.keys():
+        raise RuntimeError('Supported types: {}'.format(
+            CUPY_TO_CTYPES_MAPPING.keys()
+        ))
     addr = ctypes.cast(cptr, ctypes.c_void_p).value
     # pylint: disable=c-extension-no-member,no-member
     device = cupy.cuda.runtime.pointerGetAttributes(addr).device
     # The owner field is just used to keep the memory alive with ref count.  As
     # unowned's life time is scoped within this function we don't need that.
     unownd = UnownedMemory(
-        addr, length.value * ctypes.sizeof(ctypes.c_float), owner=None)
+        addr, length.value * ctypes.sizeof(CUPY_TO_CTYPES_MAPPING[dtype]),
+        owner=None)
     memptr = MemoryPointer(unownd, 0)
-    mem = cupy.ndarray((length.value, ), dtype=np.float32, memptr=memptr)
+    # pylint: disable=unexpected-keyword-arg
+    mem = cupy.ndarray((length.value, ), dtype=dtype, memptr=memptr)
     assert mem.device.id == device
     arr = cupy.array(mem, copy=True)
     return arr

From cfbc3cd3a5e5591ef68603e20076d41af282746c Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Wed, 15 Apr 2020 04:28:39 +0800
Subject: [PATCH 16/20] Just run all of them.

---
 tests/ci_build/test_python.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index 3c714e355166..76ff0c6e41e8 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -47,8 +47,7 @@ case "$suite" in
     install_xgboost
     pytest -v -s -rxXs --fulltrace -m "not mgpu" \
            tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py \
-	   tests/python-gpu/test_gpu_predict.py::TestGPUPredict::test_inplace_predict_cupy \
-	   tests/python-gpu/test_gpu_predict.py::TestGPUPredict::test_inplace_predict_cudf
+	   tests/python-gpu/test_gpu_predict.py
     ;;
 
   cpu)

From 07467aaf384adda5114b966532a96e5424d1d4fb Mon Sep 17 00:00:00 2001
From: fis <jm.yuan@outlook.com>
Date: Wed, 15 Apr 2020 05:09:40 +0800
Subject: [PATCH 17/20] File name.

---
 tests/ci_build/test_python.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index 76ff0c6e41e8..81c6931cb99a 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -47,7 +47,7 @@ case "$suite" in
     install_xgboost
     pytest -v -s -rxXs --fulltrace -m "not mgpu" \
            tests/python-gpu/test_from_cudf.py tests/python-gpu/test_from_cupy.py \
-	   tests/python-gpu/test_gpu_predict.py
+	   tests/python-gpu/test_gpu_prediction.py
     ;;
 
   cpu)

From bc2aa1828204baf350efe29c756c633a39f72f55 Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Tue, 14 Apr 2020 17:32:20 -0700
Subject: [PATCH 18/20] Add back cuDF tests that need multi-GPU

---
 tests/ci_build/test_python.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index 81c6931cb99a..711601152838 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -37,6 +37,11 @@ case "$suite" in
     source activate gpu_test
     install_xgboost
     pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu
+
+    source activate cudf_test
+    install_xgboost
+    pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py::test_dask_dataframe tests/python-gpu/test_gpu_with_dask.py::test_dask_array
+
     cd tests/distributed
     ./runtests-gpu.sh
     cd -

From 58ca057e7756aad859673504d9ce9d21a941c5cb Mon Sep 17 00:00:00 2001
From: Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Tue, 14 Apr 2020 17:36:42 -0700
Subject: [PATCH 19/20] Add mgpu-cudf category

---
 Jenkinsfile                   | 18 ++++++++++++------
 tests/ci_build/test_python.sh | 11 +++++++----
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a285009921e0..56a4332374ae 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -296,19 +296,25 @@ def TestPythonGPU(args) {
       sh """
       ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh mgpu
       """
+      if (args.cuda_version != '9.0') {
+        echo "Running tests with cuDF..."
+        sh """
+        ${dockerRun} cudf ${docker_binary} ${docker_args} tests/ci_build/test_python.sh mgpu-cudf
+        """
+      }
     } else {
       echo "Using a single GPU"
       sh """
       ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/test_python.sh gpu
       """
+      if (args.cuda_version != '9.0') {
+        echo "Running tests with cuDF..."
+        sh """
+        ${dockerRun} cudf ${docker_binary} ${docker_args} tests/ci_build/test_python.sh cudf
+        """
+      }
     }
     // For CUDA 10.0 target, run cuDF tests too
-    if (args.cuda_version == '10.0') {
-      echo "Running tests with cuDF..."
-      sh """
-      ${dockerRun} cudf ${docker_binary} ${docker_args} tests/ci_build/test_python.sh cudf
-      """
-    }
     deleteDir()
   }
 }
diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index 711601152838..79149c193d18 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -38,10 +38,6 @@ case "$suite" in
     install_xgboost
     pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu
 
-    source activate cudf_test
-    install_xgboost
-    pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py::test_dask_dataframe tests/python-gpu/test_gpu_with_dask.py::test_dask_array
-
     cd tests/distributed
     ./runtests-gpu.sh
     cd -
@@ -55,6 +51,13 @@ case "$suite" in
 	   tests/python-gpu/test_gpu_prediction.py
     ;;
 
+  mgpu-cudf)
+    source activate cudf_test
+    install_xgboost
+    pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py::test_dask_dataframe \
+          tests/python-gpu/test_gpu_with_dask.py::test_dask_array
+    ;;
+
   cpu)
     install_xgboost
     pytest -v -s --fulltrace tests/python

From 3dd595d80d9879250d6b69febd7525475835805b Mon Sep 17 00:00:00 2001
From: Philip Hyunsu Cho <chohyu01@cs.washington.edu>
Date: Tue, 14 Apr 2020 18:56:22 -0700
Subject: [PATCH 20/20] Just run all of test_gpu_with_dask.py

---
 tests/ci_build/test_python.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/ci_build/test_python.sh b/tests/ci_build/test_python.sh
index 79149c193d18..72c13e4cb894 100755
--- a/tests/ci_build/test_python.sh
+++ b/tests/ci_build/test_python.sh
@@ -54,8 +54,7 @@ case "$suite" in
   mgpu-cudf)
     source activate cudf_test
     install_xgboost
-    pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py::test_dask_dataframe \
-          tests/python-gpu/test_gpu_with_dask.py::test_dask_array
+    pytest -v -s -rxXs --fulltrace -m "mgpu" tests/python-gpu/test_gpu_with_dask.py
     ;;
 
   cpu)