support bootstrap allreduce/broadcast

.travis.yml

@@ -16,8 +16,7 @@ env:
     - TASK=doc
     - TASK=build
     - TASK=mpi-build
-    - TASK=cmake-build
-    - TASK=test CXX=g++
+    - TASK=cmake-test
 
 # dependent apt packages
 addons:

CMakeLists.txt

@@ -1,6 +1,14 @@
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.3)
 
-project(rabit VERSION 0.2.0)
+project(rabit VERSION 0.3.0)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+if(COMPILER_SUPPORTS_CXX11)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+else()
+  message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
+endif()
 
 option(RABIT_BUILD_TESTS "Build rabit tests" OFF)
 option(RABIT_BUILD_MPI "Build MPI" OFF)
@@ -9,10 +17,11 @@ option(RABIT_BUILD_DMLC "Include DMLC_CORE in build" ON)
 add_library(rabit src/allreduce_base.cc src/allreduce_robust.cc src/engine.cc src/c_api.cc)
 add_library(rabit_base src/allreduce_base.cc src/engine_base.cc src/c_api.cc)
 add_library(rabit_empty src/engine_empty.cc src/c_api.cc)
+add_library(rabit_mock_static src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
+add_library(rabit_mock SHARED src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
 
-set(rabit_libs rabit rabit_base rabit_empty)
-
-set_target_properties(rabit rabit_base rabit_empty PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
+set(rabit_libs rabit rabit_base rabit_empty rabit_mock rabit_mock_static)
+set_target_properties(rabit rabit_base rabit_empty rabit_mock rabit_mock_static PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
 
 if(RABIT_BUILD_MPI)
   find_package(MPI REQUIRED)
@@ -21,11 +30,6 @@ if(RABIT_BUILD_MPI)
   list(APPEND rabit_libs rabit_mpi)
 endif()
 
-if(RABIT_BUILD_TESTS)
-  # Use static so that rabit_mock won't be installed when building shared libraries.
-  add_library(rabit_mock STATIC src/allreduce_base.cc src/allreduce_robust.cc src/engine_mock.cc src/c_api.cc)
-  list(APPEND rabit_libs rabit_mock) # add to list to apply build settings, then remove
-endif()
 
 if(RABIT_BUILD_DMLC)
   foreach(lib ${rabit_libs})
@@ -46,17 +50,19 @@ else()
 endif()
 
 if(RABIT_BUILD_TESTS)
-  list(REMOVE_ITEM rabit_libs "rabit_mock") # remove here to avoid installing it
-  set(tests speed_test lazy_recover local_recover model_recover)
+  list(REMOVE_ITEM rabit_libs "rabit_mock_static") # remove here to avoid installing it
+  set(tests lazy_recover local_recover model_recover)
+
   foreach(test ${tests})
     add_executable(${test} test/${test}.cc)
-    target_link_libraries(${test} rabit_mock)
-    install(TARGETS ${test} DESTINATION bin)
+    target_link_libraries(${test} rabit_mock_static)
+    set_target_properties(${test}  PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON)
+    install(TARGETS ${test} DESTINATION test)
   endforeach()
   if(RABIT_BUILD_MPI)
     add_executable(speed_test_mpi test/speed_test.cc)
     target_link_libraries(speed_test_mpi rabit_mpi)
-    install(TARGETS speed_test_mpi DESTINATION bin)
+    install(TARGETS speed_test_mpi DESTINATION test)
   endif()
 endif()
 
@@ -66,6 +72,7 @@ endif()
 #   * <prefix>/lib/cmake/<PROJECT-NAME>
 #   * <prefix>/lib/
 #   * <prefix>/include/
+set(CMAKE_INSTALL_PREFIX "../")
 set(config_install_dir "lib/cmake/${PROJECT_NAME}")
 set(include_install_dir "include")
 

Makefile

@@ -2,61 +2,25 @@ OS := $(shell uname)
 
 RABIT_BUILD_DMLC = 0
 
-ifeq ($(RABIT_BUILD_DMLC),1)
-    DMLC=dmlc-core
-else
-    DMLC=../dmlc-core
-endif
-
 export WARNFLAGS= -Wall -Wextra -Wno-unused-parameter -Wno-unknown-pragmas -std=c++11
-export CFLAGS = -O3 $(WARNFLAGS) -I $(DMLC)/include -I include/
+export CFLAGS = -O3 $(WARNFLAGS)
 export LDFLAGS =-Llib
 
 #download mpi
 #echo $(shell scripts/mpi.sh)
 
 MPICXX=./mpich/bin/mpicxx
 
-ifeq ($(OS), Darwin)
-    ifndef CC
-        export CC = gcc-4.9
-    endif
-    ifndef CXX
-        export CXX = g++-4.9
-    endif
-else
-    ifeq ($(OS), FreeBSD)
-         ifndef CXX
-             export CXX = g++6
-         endif
-         export LDFLAGS= -Llib -Wl,-rpath=/usr/local/lib/gcc6
-    else
-        # linux defaults
-        ifndef CC
-            export CC = gcc
-        endif
-        ifndef CXX
-            export CXX = g++
-        endif
-        LDFLAGS +=-lrt
-    endif
-endif
+export CXX = g++
+
 
 #----------------------------
 # Settings for power and arm arch
 #----------------------------
 ARCH := $(shell uname -a)
-ifneq (,$(filter $(ARCH), powerpc64le ppc64le ))
-	USE_SSE=0
+ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64))
+	CFLAGS += -march=native
 else
-	USE_SSE=1
-endif
-
-ifndef USE_SSE
-	USE_SSE = 1
-endif
-
-ifeq ($(USE_SSE), 1)
 	CFLAGS += -msse2
 endif
 
@@ -71,6 +35,14 @@ ifndef LINT_LANG
 	LINT_LANG="all"
 endif
 
+ifeq ($(RABIT_BUILD_DMLC),1)
+    DMLC=dmlc-core
+else
+    DMLC=../dmlc-core
+endif
+
+CFLAGS += -I $(DMLC)/include -I include/
+
 # build path
 BPATH=.
 # objectives that makes up rabit library

include/rabit/internal/engine.h

@@ -9,6 +9,40 @@
 #include <string>
 #include "../serializable.h"
 
+// keeps rabit api caller signature
+#ifndef RABIT_API_CALLER_SIGNATURE
+#define RABIT_API_CALLER_SIGNATURE
+
+#ifdef __has_builtin
+
+#if __has_builtin(__builtin_FILE)
+#define _FILE  __builtin_FILE()
+#else
+#define _FILE  "N/A"
+#endif  // __has_builtin(__builtin_FILE)
+
+#if __has_builtin(__builtin_LINE)
+#define _LINE  __builtin_LINE()
+#else
+#define _LINE  -1
+#endif  // __has_builtin(__builtin_LINE)
+
+#if __has_builtin(__builtin_FUNCTION)
+#define _CALLER  __builtin_FUNCTION()
+#else
+#define _CALLER  "N/A"
+#endif  // __has_builtin(__builtin_FUNCTION)
+
+#else
+
+#define _FILE  "N/A"
+#define _LINE  -1
+#define _CALLER  "N/A"
+
+#endif  // __has_builtin
+
+#endif  // RABIT_API_CALLER_SIGNATURE
+
 namespace MPI {
 /*! \brief MPI data type just to be compatible with MPI reduce function*/
 class Datatype;
@@ -54,20 +88,36 @@ class IEngine {
    *                     will be called by the function before performing Allreduce in order to initialize the data in sendrecvbuf.
    *                     If the result of Allreduce can be recovered directly, then prepare_func will NOT be called
    * \param prepare_arg argument used to pass into the lazy preprocessing function
+   * \param is_bootstrap if this allreduce is needed to bootstrap failed node
+   * \param _file caller file name used to generate unique cache key
+   * \param _line caller line number used to generate unique cache key
+   * \param _caller caller function name used to generate unique cache key
    */
   virtual void Allreduce(void *sendrecvbuf_,
                          size_t type_nbytes,
                          size_t count,
                          ReduceFunction reducer,
                          PreprocFunction prepare_fun = NULL,
-                         void *prepare_arg = NULL) = 0;
+                         void *prepare_arg = NULL,
+                         bool is_bootstrap = false,
+                         const char* _file = _FILE,
+                         const int _line = _LINE,
+                         const char* _caller = _CALLER) = 0;
   /*!
    * \brief broadcasts data from root to every other node
    * \param sendrecvbuf_ buffer for both sending and receiving data
    * \param size the size of the data to be broadcasted
    * \param root the root worker id to broadcast the data
+   * \param is_bootstrap if this broadcast is needed to bootstrap failed node
+   * \param _file caller file name used to generate unique cache key
+   * \param _line caller line number used to generate unique cache key
+   * \param _caller caller function name used to generate unique cache key
    */
-  virtual void Broadcast(void *sendrecvbuf_, size_t size, int root) = 0;
+  virtual void Broadcast(void *sendrecvbuf_, size_t size, int root,
+                         bool is_bootstrap = false,
+                         const char* _file = _FILE,
+                         const int _line = _LINE,
+                         const char* _caller = _CALLER) = 0;
   /*!
    * \brief explicitly re-initialize everything before calling LoadCheckPoint
    *    call this function when IEngine throws an exception,
@@ -204,6 +254,10 @@ enum DataType {
  *                     will be called by the function before performing Allreduce, to initialize the data in sendrecvbuf_.
  *                     If the result of Allreduce can be recovered directly, then prepare_func will NOT be called
  * \param prepare_arg argument used to pass into the lazy preprocessing function.
+ * \param is_bootstrap if this allreduce is needed to bootstrap failed node
+ * \param _file caller file name used to generate unique cache key
+ * \param _line caller line number used to generate unique cache key
+ * \param _caller caller function name used to generate unique cache key
  */
 void Allreduce_(void *sendrecvbuf,
                 size_t type_nbytes,
@@ -212,8 +266,11 @@ void Allreduce_(void *sendrecvbuf,
                 mpi::DataType dtype,
                 mpi::OpType op,
                 IEngine::PreprocFunction prepare_fun = NULL,
-                void *prepare_arg = NULL);
-
+                void *prepare_arg = NULL,
+                bool is_bootstrap = false,
+                const char* _file = _FILE,
+                const int _line = _LINE,
+                const char* _caller = _CALLER);
 /*!
  * \brief handle for customized reducer, used to handle customized reduce
  *  this class is mainly created for compatiblity issues with MPI's customized reduce
@@ -239,12 +296,20 @@ class ReduceHandle {
    *                     will be called by the function before performing Allreduce in order to initialize the data in sendrecvbuf_.
    *                     If the result of Allreduce can be recovered directly, then prepare_func will NOT be called
    * \param prepare_arg argument used to pass into the lazy preprocessing function
+   * \param is_bootstrap if this allreduce is needed to bootstrap failed node
+   * \param _file caller file name used to generate unique cache key
+   * \param _line caller line number used to generate unique cache key
+   * \param _caller caller function name used to generate unique cache key
    */
   void Allreduce(void *sendrecvbuf,
                  size_t type_nbytes,
                  size_t count,
                  IEngine::PreprocFunction prepare_fun = NULL,
-                 void *prepare_arg = NULL);
+                 void *prepare_arg = NULL,
+                 bool is_bootstrap = false,
+                 const char* _file = _FILE,
+                 const int _line = _LINE,
+                 const char* _caller = _CALLER);
   /*! \return the number of bytes occupied by the type */
   static int TypeSize(const MPI::Datatype &dtype);