exit when allreduce/broadcast error cause timeout

.gitignore

@@ -47,3 +47,6 @@ mpich-3.2/
 cmake-build-debug/
 .vscode/
 
+# cmake
+build/
+

.travis.yml

@@ -8,16 +8,29 @@ osx_image: xcode10.2
 
 dist: xenial
 
+language: cpp
+
 # Use Build Matrix to do lint and build seperately
 env:
  matrix:
  - TASK=lint LINT_LANG=cpp
  - TASK=lint LINT_LANG=python
  - TASK=doc
- - TASK=build
+ # - TASK=build
  - TASK=mpi-build
  - TASK=cmake-test
 
+matrix:
+ exclude:
+ - os: osx
+ env: TASK=lint LINT_LANG=cpp
+ - os: osx
+ env: TASK=lint LINT_LANG=python
+ - os: osx
+ env: TASK=doc
+ - os: osx
+ env: TASK=build
+
 # dependent apt packages
 addons:
  apt:

CMakeLists.txt

@@ -20,11 +20,16 @@ if(R_LIB OR MINGW OR WIN32)
  CXX_STANDARD_REQUIRED ON
  POSITION_INDEPENDENT_CODE ON)
 else()
- add_library(rabit src/allreduce_base.cc src/allreduce_robust.cc src/engine.cc src/c_api.cc)
- add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
+ find_package(Threads REQUIRED)
  add_library(rabit_empty src/engine_empty.cc src/c_api.cc)
+ add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
+
+ add_library(rabit src/allreduce_base.cc src/allreduce_robust.cc src/engine.cc src/c_api.cc)
  add_library(rabit_mock_static src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
  add_library(rabit_mock SHARED src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
+ target_link_libraries(rabit Threads::Threads)
+ target_link_libraries(rabit_mock_static Threads::Threads)
+ target_link_libraries(rabit_mock Threads::Threads)
 
  set(rabit_libs rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
  set_target_properties(rabit rabit_base rabit_empty rabit_mock rabit_mock_static

include/rabit/internal/utils.h

@@ -86,7 +86,7 @@ inline void HandleAssertError(const char *msg) {
  */
 inline void HandleCheckError(const char *msg) {
  if (STOP_PROCESS_ON_ERROR) {
- fprintf(stderr, "%s, shutting down process", msg);
+ fprintf(stderr, "%s, shutting down process\n", msg);
  exit(-1);
  } else {
  fprintf(stderr, "%s, rabit is configured to keep process running\n", msg);

scripts/travis_script.sh

@@ -25,9 +25,12 @@ if [ ${TASK} == "cmake-test" ]; then
  mkdir build
  cd build
  cmake -DRABIT_BUILD_TESTS=ON -DRABIT_BUILD_DMLC=ON -DGTEST_ROOT=${HOME}/.local ..
+ # known osx gtest 1.8 issue
+ cp ${HOME}/.local/lib/*.dylib .
  #unit tests
- make
+ make -j4
  make test
+ #make test VERBOSE=1 || exit -1
  make install || exit -1
  cd ../test
  ../scripts/travis_runtest.sh || exit -1

src/allreduce_base.cc

@@ -214,6 +214,13 @@ void AllreduceBase::SetParam(const char *name, const char *val) {
  if (!strcmp(name, "rabit_debug")) {
  rabit_debug = atoi(val);
  }
+ if (!strcmp(name, "rabit_timeout")) {
+ rabit_timeout = atoi(val);
+ }
+ if (!strcmp(name, "rabit_timeout_sec")) {
+ timeout_sec = atoi(val);
+ utils::Assert(rabit_timeout > 0, "rabit_timeout_sec should be greater than 0 second");
+ }
 }
 /*!
  * \brief initialize connection to the tracker

src/allreduce_base.h

@@ -546,6 +546,10 @@ class AllreduceBase : public IEngine {
  int rabit_bootstrap_cache = 0;
  // enable detailed logging
  int rabit_debug = 0;
+ // by default, if rabit worker not recover in half an hour exit
+ int timeout_sec = 1800;
+ // flag to enable rabit_timeout
+ int rabit_timeout = 0;
 };
 } // namespace engine
 } // namespace rabit

src/allreduce_mock.h

@@ -176,7 +176,7 @@ class AllreduceMock : public AllreduceRobust {
  if (mock_map.count(key) != 0) {
  num_trial += 1;
  // data processing frameworks runs on shared process
- utils::Error("[%d]@@@Hit Mock Error:%s\n", rank, name);
+ _error("[%d]@@@Hit Mock Error:%s ", rank, name);
  }
  }
 };