Revert "[jvm-packages] update rabit, surface new changes to spark, ad…

…d parity and failure tests (#4876)" (#4965)

This reverts commit 86ed01c.

CMakeLists.txt

@@ -94,16 +94,36 @@ set_target_properties(dmlc PROPERTIES
 list(APPEND LINKED_LIBRARIES_PRIVATE dmlc)
 
 # rabit
-set(RABIT_BUILD_DMLC OFF)
-set(DMLC_ROOT ${xgboost_SOURCE_DIR}/dmlc-core)
-set(RABIT_WITH_R_LIB ${R_LIB})
-add_subdirectory(rabit)
-
-if (RABIT_MOCK)
-  list(APPEND LINKED_LIBRARIES_PRIVATE rabit_mock_static)
-else()
-  list(APPEND LINKED_LIBRARIES_PRIVATE rabit)
-endif(RABIT_MOCK)
+# full rabit doesn't build on windows, so we can't import it as subdirectory
+if(MINGW OR R_LIB OR WIN32)
+  set(RABIT_SOURCES
+    rabit/src/engine_empty.cc
+    rabit/src/c_api.cc)
+else ()
+  if(RABIT_MOCK)
+    set(RABIT_SOURCES
+      rabit/src/allreduce_base.cc
+      rabit/src/allreduce_robust.cc
+      rabit/src/engine_mock.cc
+      rabit/src/c_api.cc)
+  else()
+    set(RABIT_SOURCES
+      rabit/src/allreduce_base.cc
+      rabit/src/allreduce_robust.cc
+      rabit/src/engine.cc
+      rabit/src/c_api.cc)
+  endif(RABIT_MOCK)
+endif (MINGW OR R_LIB OR WIN32)
+add_library(rabit STATIC ${RABIT_SOURCES})
+target_include_directories(rabit PRIVATE
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/dmlc-core/include>
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/rabit/include/rabit>)
+set_target_properties(rabit
+  PROPERTIES
+  CXX_STANDARD 11
+  CXX_STANDARD_REQUIRED ON
+  POSITION_INDEPENDENT_CODE ON)
+list(APPEND LINKED_LIBRARIES_PRIVATE rabit)
 
 # Exports some R specific definitions and objects
 if (R_LIB)

jvm-packages/.gitignore

@@ -1,3 +1,2 @@
 tracker.py
-build.sh
-
+build.sh

jvm-packages/create_jni.py

@@ -18,7 +18,7 @@
     "USE_HDFS": "OFF",
     "USE_AZURE": "OFF",
     "USE_S3": "OFF",
-    "RABIT_MOCK": "OFF",
+
     "USE_CUDA": "OFF",
     "JVM_BINDINGS": "ON"
 }
@@ -68,7 +68,6 @@ def normpath(path):
 
 
 if __name__ == "__main__":
-    CONFIG["RABIT_MOCK"] = str(sys.argv[1])
     if sys.platform == "darwin":
         # Enable of your compiler supports OpenMP.
         CONFIG["USE_OPENMP"] = "OFF"

jvm-packages/pom.xml

@@ -37,7 +37,6 @@
         <spark.version>2.4.3</spark.version>
         <scala.version>2.12.8</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
-        <rabit.mock>OFF</rabit.mock>
     </properties>
     <repositories>
         <repository>
@@ -53,47 +52,6 @@
         <module>xgboost4j-flink</module>
     </modules>
     <profiles>
-        <profile>
-          <id>dev</id>
-          <properties>
-            <rabit.mock>ON</rabit.mock>
-          </properties>
-          <build>
-            <plugins>
-              <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <version>3.0.2</version>
-                <executions>
-                  <execution>
-                    <id>empty-javadoc-jar</id>
-                    <phase>package</phase>
-                    <goals>
-                      <goal>jar</goal>
-                    </goals>
-                    <configuration>
-                      <classifier>javadoc</classifier>
-                      <classesDirectory>${basedir}/javadoc</classesDirectory>
-                    </configuration>
-                  </execution>
-                </executions>
-              </plugin>
-              <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
-                <version>2.2.1</version>
-                <executions>
-                  <execution>
-                    <id>attach-sources</id>
-                    <goals>
-                      <goal>jar-no-fork</goal>
-                    </goals>
-                  </execution>
-                </executions>
-              </plugin>
-            </plugins>
-          </build>
-        </profile>
         <profile>
             <id>release</id>
             <build>

jvm-packages/scalastyle-config.xml

@@ -49,7 +49,7 @@ This file is divided into 3 sections:
   <check level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
     <parameters>
        <parameter name="header"><![CDATA[/*
-				  Copyright (c) 2014 - 2019 by Contributors
+				  Copyright (c) 2014 by Contributors
 
 					 Licensed under the Apache License, Version 2.0 (the "License");
 					  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/BasicWalkThrough.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/BoostFromPrediction.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/CrossValidation.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/CustomObjective.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/ExternalMemory.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/GeneralizedLinearModel.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/PredictFirstNTree.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/PredictLeafIndices.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/flink/DistTrainWithFlink.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/SparkMLlibPipeline.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/spark/SparkTraining.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/util/CustomEval.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-flink/src/main/scala/ml/dmlc/xgboost4j/scala/flink/XGBoost.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-flink/src/main/scala/ml/dmlc/xgboost4j/scala/flink/XGBoostModel.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/CheckpointManager.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/DataUtils.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoost.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -21,11 +21,11 @@ import java.nio.file.Files
 
 import scala.collection.{AbstractIterator, mutable}
 import scala.util.Random
-import scala.collection.JavaConverters._
 
 import ml.dmlc.xgboost4j.java.{IRabitTracker, Rabit, XGBoostError, RabitTracker => PyRabitTracker}
 import ml.dmlc.xgboost4j.scala.rabit.RabitTracker
 import ml.dmlc.xgboost4j.scala.spark.CheckpointManager.CheckpointParam
+import ml.dmlc.xgboost4j.scala.spark.XGBoost.logger
 import ml.dmlc.xgboost4j.scala.spark.params.LearningTaskParams
 import ml.dmlc.xgboost4j.scala.{XGBoost => SXGBoost, _}
 import ml.dmlc.xgboost4j.{LabeledPoint => XGBLabeledPoint}
@@ -155,7 +155,7 @@ private[this] class XGBoostExecutionParamsFactory(rawParams: Map[String, Any], s
     overridedParams
   }
 
-  private[spark] def buildXGBRuntimeParams: XGBoostExecutionParams = {
+  def buildXGBRuntimeParams: XGBoostExecutionParams = {
     val nWorkers = overridedParams("num_workers").asInstanceOf[Int]
     val round = overridedParams("num_round").asInstanceOf[Int]
     val useExternalMemory = overridedParams("use_external_memory").asInstanceOf[Boolean]
@@ -221,25 +221,6 @@ private[this] class XGBoostExecutionParamsFactory(rawParams: Map[String, Any], s
     xgbExecParam.setRawParamMap(overridedParams)
     xgbExecParam
   }
-
-  private[spark] def buildRabitParams: Map[String, String] = Map(
-    "rabit_reduce_ring_mincount" ->
-      overridedParams.getOrElse("rabit_reduce_ring_mincount", 32<<10).toString,
-    "rabit_reduce_buffer" ->
-      overridedParams.getOrElse("rabit_reduce_buffer", "256MB").toString,
-    "rabit_bootstrap_cache" ->
-      overridedParams.getOrElse("rabit_bootstrap_cache", false).toString,
-    "rabit_debug" ->
-      overridedParams.getOrElse("rabit_debug", false).toString,
-    "rabit_timeout" ->
-      overridedParams.getOrElse("rabit_timeout", false).toString,
-    "rabit_timeout_sec" ->
-      overridedParams.getOrElse("rabit_timeout_sec", 1800).toString,
-    "DMLC_WORKER_CONNECT_RETRY" ->
-      overridedParams.getOrElse("dmlc_worker_connect_retry", 5).toString,
-    "DMLC_WORKER_STOP_PROCESS_ON_ERROR" ->
-      overridedParams.getOrElse("dmlc_worker_stop_process_on_error", false).toString
-  )
 }
 
 /**
@@ -340,6 +321,7 @@ object XGBoost extends Serializable {
     }
     val taskId = TaskContext.getPartitionId().toString
     rabitEnv.put("DMLC_TASK_ID", taskId)
+    rabitEnv.put("DMLC_WORKER_STOP_PROCESS_ON_ERROR", "false")
 
     try {
       Rabit.init(rabitEnv)
@@ -508,9 +490,8 @@ object XGBoost extends Serializable {
       evalSetsMap: Map[String, RDD[XGBLabeledPoint]] = Map()):
     (Booster, Map[String, Array[Float]]) = {
     logger.info(s"Running XGBoost ${spark.VERSION} with parameters:\n${params.mkString("\n")}")
-    val xgbParamsFactory = new XGBoostExecutionParamsFactory(params, trainingData.sparkContext)
-    val xgbExecParams = xgbParamsFactory.buildXGBRuntimeParams
-    val xgbRabitParams = xgbParamsFactory.buildRabitParams
+    val xgbExecParams = new XGBoostExecutionParamsFactory(params, trainingData.sparkContext).
+      buildXGBRuntimeParams
     val sc = trainingData.sparkContext
     val checkpointManager = new CheckpointManager(sc, xgbExecParams.checkpointParam.
       checkpointPath)
@@ -529,12 +510,12 @@ object XGBoost extends Serializable {
             val parallelismTracker = new SparkParallelismTracker(sc,
               xgbExecParams.timeoutRequestWorkers,
               xgbExecParams.numWorkers)
-            val rabitEnv = tracker.getWorkerEnvs.asScala ++ xgbRabitParams
+            val rabitEnv = tracker.getWorkerEnvs
             val boostersAndMetrics = if (hasGroup) {
-              trainForRanking(transformedTrainingData.left.get, xgbExecParams, rabitEnv.asJava,
+              trainForRanking(transformedTrainingData.left.get, xgbExecParams, rabitEnv,
                 checkpointRound, prevBooster, evalSetsMap)
             } else {
-              trainForNonRanking(transformedTrainingData.right.get, xgbExecParams, rabitEnv.asJava,
+              trainForNonRanking(transformedTrainingData.right.get, xgbExecParams, rabitEnv,
                 checkpointRound, prevBooster, evalSetsMap)
             }
             val sparkJobThread = new Thread() {

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostClassifier.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ class XGBoostClassifier (
   def this(xgboostParams: Map[String, Any]) = this(
     Identifiable.randomUID("xgbc"), xgboostParams)
 
-  XGBoost2MLlibParams(xgboostParams)
+  XGBoostToMLlibParams(xgboostParams)
 
   def setWeightCol(value: String): this.type = set(weightCol, value)
 

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostEstimatorCommon.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -21,7 +21,7 @@ import ml.dmlc.xgboost4j.scala.spark.params._
 import org.apache.spark.ml.param.shared.HasWeightCol
 
 private[spark] sealed trait XGBoostEstimatorCommon extends GeneralParams with LearningTaskParams
-  with BoosterParams with RabitParams with ParamMapFuncs with NonParamVariables {
+  with BoosterParams with ParamMapFuncs with NonParamVariables {
 
   def needDeterministicRepartitioning: Boolean = {
     getCheckpointPath.nonEmpty && getCheckpointInterval > 0

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostRegressor.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -54,7 +54,7 @@ class XGBoostRegressor (
   def this(xgboostParams: Map[String, Any]) = this(
     Identifiable.randomUID("xgbr"), xgboostParams)
 
-  XGBoost2MLlibParams(xgboostParams)
+  XGBoostToMLlibParams(xgboostParams)
 
   def setWeightCol(value: String): this.type = set(weightCol, value)
 

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoostTrainingSummary.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.

jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/package.scala

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 - 2019 by Contributors
+ Copyright (c) 2014 by Contributors
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.