[WIP] [CI] Use Ninja to speed up builds

.travis.yml

@@ -46,7 +46,17 @@ addons:
  - lz4
  - wget
  - r
+ - ninja
  update: true
+ apt:
+ sources:
+ - sourceline: 'deb https://apt.kitware.com/ubuntu/ bionic main'
+ key_url: 'https://apt.kitware.com/keys/kitware-archive-latest.asc'
+ packages:
+ - cmake
+ - build-essential
+ - wget
+ - ninja-build
 
 before_install:
  - source tests/travis/travis_setup_env.sh

Jenkinsfile

@@ -190,7 +190,7 @@ def BuildCPU() {
  # This step is not necessary, but here we include it, to ensure that DMLC_CORE_USE_CMAKE flag is correctly propagated
  # We want to make sure that we use the configured header build/dmlc/build_config.h instead of include/dmlc/build_config_default.h.
  # See discussion at https://github.com/dmlc/xgboost/issues/5510
- ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON
+ ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON -DCMAKE_BUILD_TYPE=Release
  ${dockerRun} ${container_type} ${docker_binary} build/testxgboost
  """
  // Sanitizer test
@@ -228,7 +228,7 @@ def BuildCPUNonOmp() {
  def container_type = "cpu"
  def docker_binary = "docker"
  sh """
- ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_OPENMP=OFF
+ ${dockerRun} ${container_type} ${docker_binary} tests/ci_build/build_via_cmake.sh -DUSE_OPENMP=OFF -DCMAKE_BUILD_TYPE=Release
  """
  echo "Running Non-OpenMP C++ test..."
  sh """
@@ -246,7 +246,7 @@ def BuildCUDA(args) {
  def docker_binary = "docker"
  def docker_args = "--build-arg CUDA_VERSION=${args.cuda_version}"
  sh """
- ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON
+ ${dockerRun} ${container_type} ${docker_binary} ${docker_args} tests/ci_build/build_via_cmake.sh -DUSE_CUDA=ON -DUSE_NCCL=ON -DOPEN_MP:BOOL=ON -DHIDE_CXX_SYMBOLS=ON -DCMAKE_BUILD_TYPE=Release
  ${dockerRun} ${container_type} ${docker_binary} ${docker_args} bash -c "cd python-package && rm -rf dist/* && python setup.py bdist_wheel --universal"
  ${dockerRun} ${container_type} ${docker_binary} ${docker_args} python3 tests/ci_build/rename_whl.py python-package/dist/*.whl ${commit_id} manylinux2010_x86_64
  """

Jenkinsfile-win64

@@ -74,11 +74,11 @@ def BuildWin64() {
  bat """
  mkdir build
  cd build
- cmake .. -G"Visual Studio 15 2017 Win64" -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON
+ "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat" && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON
  """
  bat """
  cd build
- "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\MSBuild.exe" xgboost.sln /m /p:Configuration=Release /nodeReuse:false
+ "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvars64.bat" && ninja -v
  """
  bat """
  cd python-package

appveyor.yml

@@ -6,17 +6,20 @@ environment:
  ver: 2015
  generator: "Visual Studio 14 2015 Win64"
  configuration: Debug
+ vcvars: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat
  - target: msvc
  ver: 2015
  generator: "Visual Studio 14 2015 Win64"
  configuration: Release
+ vcvars: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat
  - target: mingw
  generator: "Unix Makefiles"
  - target: jvm
  - target: rmsvc
  ver: 2015
  generator: "Visual Studio 14 2015 Win64"
  configuration: Release
+ vcvars: C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat
  - target: rmingw
  generator: "Unix Makefiles"
 
@@ -65,14 +68,15 @@ build_script:
  - if /i "%target%" == "msvc" (
  mkdir build_msvc%ver% &&
  cd build_msvc%ver% &&
- cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release;Debug;" &&
- msbuild xgboost.sln
+ call "%vcvars%" &&
+ cmake .. -GNinja -DCMAKE_BUILD_TYPE=%configuration% -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" &&
+ ninja -v
  )
  - if /i "%target%" == "mingw" (
  mkdir build_mingw &&
  cd build_mingw &&
- cmake .. -G"%generator%" &&
- make -j2
+ cmake .. -GNinja &&
+ ninja -v
  )
  # Python package
  - if /i "%DO_PYTHON%" == "on" (
@@ -91,7 +95,8 @@ build_script:
  - if /i "%target%" == "rmsvc" (
  mkdir build_rmsvc%ver% &&
  cd build_rmsvc%ver% &&
- cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release" -DR_LIB=ON &&
+ call "%vcvars%" &&
+ cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DR_LIB=ON -DCMAKE_C_COMPILER="cl.exe" -DCMAKE_CXX_COMPILER="cl.exe" &&
  cmake --build . --target install --config Release
  )
  - if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j_2.12

jvm-packages/create_jni.py

@@ -81,20 +81,24 @@ def normpath(path):
  if sys.platform == "win32":
  # Force x64 build on Windows.
  maybe_generator = ' -G"Visual Studio 14 Win64"'
+ maybe_parallel_build = ""
+ elif getattr(shutil, 'which', None) is not None and shutil.which('ninja'):
+ maybe_generator = ' -GNinja'
+ maybe_parallel_build = " -- -v"
  else:
  maybe_generator = ""
- if sys.platform == "linux":
- maybe_parallel_build = " -- -j $(nproc)"
- else:
- maybe_parallel_build = ""
+  if sys.platform == "linux":
+  maybe_parallel_build = " -- -j $(nproc)"
+  else:
+  maybe_parallel_build = ""
 
  args = ["-D{0}:BOOL={1}".format(k, v) for k, v in CONFIG.items()]
 
  # if enviorment set rabit_mock
  if os.getenv("RABIT_MOCK", None) is not None:
  args.append("-DRABIT_MOCK:BOOL=ON")
 
- run("cmake .. " + " ".join(args) + maybe_generator)
+ run("cmake .. -DCMAKE_BUILD_TYPE=Release " + " ".join(args) + maybe_generator)
  run("cmake --build . --config Release" + maybe_parallel_build)
 
  with cd("demo/regression"):

tests/ci_build/Dockerfile.cpu

@@ -7,7 +7,7 @@ SHELL ["/bin/bash", "-c"] # Use Bash as shell
 # Install all basic requirements
 RUN \
  apt-get update && \
- apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 liblz4-dev && \
+ apt-get install -y tar unzip wget git build-essential doxygen graphviz llvm libasan2 libidn11 ninja-build liblz4-dev && \
  # CMake
  wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
  bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \

tests/ci_build/Dockerfile.gpu_build

@@ -21,7 +21,14 @@ RUN \
  bash Miniconda3.sh -b -p /opt/python && \
  # CMake
  wget -nv -nc https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.sh --no-check-certificate && \
- bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr
+ bash cmake-3.13.0-Linux-x86_64.sh --skip-license --prefix=/usr && \
+ # Ninja
+ mkdir -p /usr/local && \
+ cd /usr/local/ && \
+ wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
+ tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
+ cd ninja && \
+ python ./configure.py --bootstrap
 
 # NCCL2 (License: https://docs.nvidia.com/deeplearning/sdk/nccl-sla/index.html)
 RUN \
@@ -33,7 +40,7 @@ RUN \
  yum install -y libnccl-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-devel-${NCCL_VERSION}+cuda${CUDA_SHORT} libnccl-static-${NCCL_VERSION}+cuda${CUDA_SHORT} && \
  rm -f nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm;
 
-ENV PATH=/opt/python/bin:$PATH
+ENV PATH=/opt/python/bin:/usr/local/ninja:$PATH
 ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
 ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp

tests/ci_build/Dockerfile.jvm

@@ -24,11 +24,20 @@ RUN \
  tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \
  ln -s /opt/apache-maven-3.6.1/ /opt/maven
 
-ENV PATH=/opt/python/bin:/opt/maven/bin:$PATH
+ENV PATH=/opt/python/bin:/opt/maven/bin:/usr/local/ninja:$PATH
 ENV CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
 ENV CXX=/opt/rh/devtoolset-4/root/usr/bin/c++
 ENV CPP=/opt/rh/devtoolset-4/root/usr/bin/cpp
 
+# Install Ninja
+RUN \
+ mkdir -p /usr/local && \
+ cd /usr/local/ && \
+ wget -nv -nc https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz --no-check-certificate && \
+ tar xf v1.10.0.tar.gz && mv ninja-1.10.0 ninja && rm -v v1.10.0.tar.gz && \
+ cd ninja && \
+ python ./configure.py --bootstrap
+
 # Install Python packages
 RUN \
  pip install numpy pytest scipy scikit-learn wheel kubernetes urllib3==1.22 awscli

tests/ci_build/build_mock_cmake.sh

@@ -4,7 +4,6 @@ set -e
 rm -rf build
 mkdir build
 cd build
-cmake -DRABIT_MOCK=ON -DCMAKE_VERBOSE_MAKEFILE=ON ..
-make clean
-make -j$(nproc)
+cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DRABIT_MOCK=ON -DCMAKE_VERBOSE_MAKEFILE=ON ..
+ninja -v
 cd ..

tests/ci_build/build_via_cmake.sh

@@ -4,7 +4,6 @@ set -e
 rm -rf build
 mkdir build
 cd build
-cmake .. "$@" -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON
-make clean
-make -j$(nproc)
+cmake .. "$@" -GNinja -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+ninja -v
 cd ..

tests/ci_build/doxygen.sh

@@ -13,7 +13,7 @@ branch_name=$1
 rm -rf build
 mkdir build
 cd build
-cmake .. -DBUILD_C_DOC=ON
-make -j
+cmake .. -GNinja -DBUILD_C_DOC=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+ninja -v
 
 tar cvjf ${branch_name}.tar.bz2 doc_doxygen/

tests/cpp/predictor/test_gpu_predictor.cu

@@ -121,6 +121,7 @@ TEST(GPUPredictor, ExternalMemoryTest) {
  ASSERT_EQ(host_vector[i * n_classes + 2], 0.5);
  }
  }
+ dmats.clear();
 }
 
 TEST(GPUPredictor, InplacePredictCupy) {

tests/travis/run_test.sh

@@ -21,8 +21,8 @@ if [ ${TASK} == "python_test" ]; then
  # Build/test
  rm -rf build
  mkdir build && cd build
- cmake .. -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
- make -j$(nproc)
+ cmake .. -GNinja -DUSE_OPENMP=ON -DCMAKE_VERBOSE_MAKEFILE=ON
+ ninja -v
 
  echo "-------------------------------"
  conda activate python3
@@ -82,8 +82,8 @@ if [ ${TASK} == "cmake_test" ]; then
  rm -rf build
  mkdir build && cd build
  PLUGINS="-DPLUGIN_LZ4=ON -DPLUGIN_DENSE_PARSER=ON"
- cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON ${PLUGINS}
- make -j$(nproc)
+ cmake .. -GNinja -DCMAKE_VERBOSE_MAKEFILE=ON -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON ${PLUGINS}
+ ninja -v
  ./testxgboost
  cd ..
  rm -rf build