diff --git a/.travis.yml b/.travis.yml
index d0f72423b6ea..6589a6fe1c01 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,7 +46,17 @@ addons:
       - lz4
       - wget
       - r
+      - ninja
     update: true
+  apt:
+    sources:
+      - sourceline: 'deb https://apt.kitware.com/ubuntu/ bionic main'
+        key_url: 'https://apt.kitware.com/keys/kitware-archive-latest.asc'
+    packages:
+      - cmake
+      - build-essential
+      - wget
+      - ninja-build
 
 before_install:
   - source tests/travis/travis_setup_env.sh
diff --git a/Jenkinsfile b/Jenkinsfile
index 9aacc64123e1..91bcacd24b32 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -190,7 +190,7 @@ def BuildCPU() {
       # This step is not necessary, but here we include it, to ensure that DMLC_CORE_USE_CMAKE flag is correctly propagated
       # We want to make sure that we use the configured header build/dmlc/build_config.h instead of include/dmlc/build_config_default.h.
       # See discussion at https://github.com/dmlc/xgboost/issues/5510
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON -DCMAKE_BUILD_TYPE=Release
     ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
     """
     // Sanitizer test
@@ -228,7 +228,7 @@ def BuildCPUNonOmp() {
     def container_type = "cpu"
     def docker_binary = "docker"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_OPENMP=OFF
+    ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_OPENMP=OFF -DCMAKE_BUILD_TYPE=Release
     """
     echo "Running Non-OpenMP C++ test..."
     sh """
@@ -246,7 +246,7 @@ def BuildCUDA(args) {
     def docker_binary = "docker"
     def docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}"
     sh """
-    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON
+    ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON -DCMAKE_BUILD_TYPE=Release
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
     ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64
     """
diff --git a/Jenkinsfile-win64 b/Jenkinsfile-win64
index 9cd12cf4f134..ca003260d5ad 100644
--- a/Jenkinsfile-win64
+++ b/Jenkinsfile-win64
@@ -74,11 +74,11 @@ def BuildWin64() {
     bat """
     mkdir build
     cd build
-    cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON
+    "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat" && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON
     """
     bat """
     cd build
-    "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release /nodeReuse:false
+    "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat" && ninja -v
     """
     bat """
     cd python-package
diff --git a/appveyor.yml b/appveyor.yml
index 54032bcc14cc..e3cb95ceff4c 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -6,10 +6,12 @@ environment:
           ver: 2015
           generator: "Visual Studio 14 2015 Win64"
           configuration: Debug
+          vcvars: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat
         - target: msvc
           ver: 2015
           generator: "Visual Studio 14 2015 Win64"
           configuration: Release
+          vcvars: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat
         - target: mingw
           generator: "Unix Makefiles"
         - target: jvm
@@ -17,6 +19,7 @@ environment:
           ver: 2015
           generator: "Visual Studio 14 2015 Win64"
           configuration: Release
+          vcvars: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat
         - target: rmingw
           generator: "Unix Makefiles"
 
@@ -65,14 +68,15 @@ build_script:
     - if /i "%target%" == "msvc" (
         mkdir build_msvc%ver% &&
         cd build_msvc%ver% &&
-        cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release;Debug;" &&
-        msbuild xgboost.sln
+        call "%vcvars%" &&
+        cmake .. -GNinja -DCMAKE_BUILD_TYPE=%configuration% -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" &&
+        ninja -v
       )
     - if /i "%target%" == "mingw" (
         mkdir build_mingw &&
         cd build_mingw &&
-        cmake .. -G"%generator%" &&
-        make -j2
+        cmake .. -GNinja &&
+        ninja -v
       )
     # Python package
     - if /i "%DO_PYTHON%" == "on" (
@@ -91,7 +95,8 @@ build_script:
     - if /i "%target%" == "rmsvc" (
         mkdir build_rmsvc%ver% &&
         cd build_rmsvc%ver% &&
-        cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release" -DR_LIB=ON &&
+        call "%vcvars%" &&
+        cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DR_LIB=ON  -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" &&
         cmake --build . --target install --config Release
       )
     - if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j_2.12
diff --git a/jvm-packages/create_jni.py b/jvm-packages/create_jni.py
index 11c33d5846b6..156813858fcf 100755
--- a/jvm-packages/create_jni.py
+++ b/jvm-packages/create_jni.py
@@ -81,12 +81,16 @@ def normpath(path):
             if sys.platform == "win32":
                 # Force x64 build on Windows.
                 maybe_generator = ' -G"Visual Studio 14 Win64"'
+                maybe_parallel_build = ""
+            elif getattr(shutil, 'which', None) is not None and shutil.which('ninja'):
+                maybe_generator = ' -GNinja'
+                maybe_parallel_build = " -- -v"
             else:
                 maybe_generator = ""
-            if sys.platform == "linux":
-                maybe_parallel_build = " -- -j $(nproc)"
-            else:
-                maybe_parallel_build = ""
+                if sys.platform == "linux":
+                    maybe_parallel_build = " -- -j $(nproc)"
+                else:
+                    maybe_parallel_build = ""
 
             args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()]
 
@@ -94,7 +98,7 @@ def normpath(path):
             if os.getenv("RABIT_MOCK", None) is not None:
                 args.append("-DRABIT_MOCK:BOOL=ON")
 
-            run("cmake .. " + " ".join(args) + maybe_generator)
+            run("cmake .. -DCMAKE_BUILD_TYPE=Release " + " ".join(args) + maybe_generator)
             run("cmake --build . --config Release" + maybe_parallel_build)
 
         with cd("demo/regression"):
diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
index 1dd8a9da16ba..d97aafbf52ec 100644
--- a/tests/ci_build/Dockerfile.cpu
+++ b/tests/ci_build/Dockerfile.cpu
@@ -7,7 +7,7 @@ SHELL ["/bin/bash", "-c"]   # Use Bash as shell
 # Install all basic requirements
 RUN \
     apt-get update && \
-    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 liblz4-dev && \
+    apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build liblz4-dev && \
     # CMake
     wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
     bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
diff --git a/tests/ci_build/Dockerfile.gpu_build b/tests/ci_build/Dockerfile.gpu_build
index 65bccce74cde..8a614db81ceb 100644
--- a/tests/ci_build/Dockerfile.gpu_build
+++ b/tests/ci_build/Dockerfile.gpu_build
@@ -21,7 +21,14 @@ RUN \
     bash Miniconda3.sh -b -p /opt/python && \
     # CMake
     wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
-    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr
+    bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Ninja
+    mkdir -p /usr/local && \
+    cd /usr/local/ && \
+    wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
+    tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
+    cd ninja && \
+    python ./configure.py --bootstrap
 
 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
@@ -33,7 +40,7 @@ RUN \
     yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
     rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
 
-ENV PATH=/opt/python/bin:$PATH
+ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH
 ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
 ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
diff --git a/tests/ci_build/Dockerfile.jvm b/tests/ci_build/Dockerfile.jvm
index b161821024d0..803795ef8f22 100644
--- a/tests/ci_build/Dockerfile.jvm
+++ b/tests/ci_build/Dockerfile.jvm
@@ -24,11 +24,20 @@ RUN \
     tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
     ln -s /opt/apache-maven-3.6.1/ /opt/maven
 
-ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
+ENV PATH=/opt/python/bin:/opt/maven/bin:/usr/local/ninja:$PATH
 ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
 ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
 
+# Install Ninja
+RUN \
+    mkdir -p /usr/local && \
+    cd /usr/local/ && \
+    wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
+    tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
+    cd ninja && \
+    python ./configure.py --bootstrap
+
 # Install Python packages
 RUN \
     pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli
diff --git a/tests/ci_build/build_mock_cmake.sh b/tests/ci_build/build_mock_cmake.sh
index 8cbabd036d97..dfc26fe2cc88 100755
--- a/tests/ci_build/build_mock_cmake.sh
+++ b/tests/ci_build/build_mock_cmake.sh
@@ -4,7 +4,6 @@ set -e
 rm -rf build
 mkdir build
 cd build
-cmake -DRABIT_MOCK=ON -DCMAKE_VERBOSE_MAKEFILE=ON ..
-make clean
-make -j$(nproc)
+cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DRABIT_MOCK=ON -DCMAKE_VERBOSE_MAKEFILE=ON ..
+ninja -v
 cd ..
diff --git a/tests/ci_build/build_via_cmake.sh b/tests/ci_build/build_via_cmake.sh
index 98808141b0e8..6d7bef778777 100755
--- a/tests/ci_build/build_via_cmake.sh
+++ b/tests/ci_build/build_via_cmake.sh
@@ -4,7 +4,6 @@ set -e
 rm -rf build
 mkdir build
 cd build
-cmake .. "$@" -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON
-make clean
-make -j$(nproc)
+cmake .. "$@" -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+ninja -v
 cd ..
diff --git a/tests/ci_build/doxygen.sh b/tests/ci_build/doxygen.sh
index 41757eb6935f..feaca063a8d2 100755
--- a/tests/ci_build/doxygen.sh
+++ b/tests/ci_build/doxygen.sh
@@ -13,7 +13,7 @@ branch_name=$1
 rm -rf build
 mkdir build
 cd build
-cmake .. -DBUILD_C_DOC=ON
-make -j
+cmake .. -GNinja -DBUILD_C_DOC=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+ninja -v
 
 tar cvjf ${branch_name}.tar.bz2 doc_doxygen/
diff --git a/tests/cpp/predictor/test_gpu_predictor.cu b/tests/cpp/predictor/test_gpu_predictor.cu
index fc40d1778b84..348851abc615 100644
--- a/tests/cpp/predictor/test_gpu_predictor.cu
+++ b/tests/cpp/predictor/test_gpu_predictor.cu
@@ -121,6 +121,7 @@ TEST(GPUPredictor, ExternalMemoryTest) {
       ASSERT_EQ(host_vector[i * n_classes + 2], 0.5);
     }
   }
+  dmats.clear();
 }
 
 TEST(GPUPredictor, InplacePredictCupy) {
diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh
index a0e1c9f28651..3e1dafebd0ef 100755
--- a/tests/travis/run_test.sh
+++ b/tests/travis/run_test.sh
@@ -21,8 +21,8 @@ if [ ${TASK} == "python_test" ]; then
     # Build/test
     rm -rf build
     mkdir build && cd build
-    cmake .. -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
-    make -j$(nproc)
+    cmake .. -GNinja -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+    ninja -v
 
     echo "-------------------------------"
     conda activate python3
@@ -82,8 +82,8 @@ if [ ${TASK} == "cmake_test" ]; then
     rm -rf build
     mkdir build && cd build
     PLUGINS="-DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON"
-    cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON ${PLUGINS}
-    make -j$(nproc)
+    cmake .. -GNinja -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON ${PLUGINS}
+    ninja -v
     ./testxgboost
     cd ..
     rm -rf build