From ea8ca6c3bf4d57038c2d4166fd2a2647b84b3a8e Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Sat, 15 Jun 2019 23:03:23 -0400 Subject: [PATCH 01/11] bump scala to 2.12 which requires java 8 and also newer flink and akka --- jvm-packages/pom.xml | 12 ++++++------ jvm-packages/xgboost4j/pom.xml | 4 ++-- .../ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala | 10 +++------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml index 1f3b0ca0f001..5401b9cbd084 100644 --- a/jvm-packages/pom.xml +++ b/jvm-packages/pom.xml @@ -31,12 +31,12 @@ UTF-8 UTF-8 - 1.7 - 1.7 - 1.5.0 + 1.8 + 1.8 + 1.7.2 2.4.3 - 2.11.12 - 2.11 + 2.12.8 + 2.12 @@ -372,7 +372,7 @@ org.scalatest scalatest_${scala.binary.version} - 3.0.0 + 3.0.8 test diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index 66045b2a081c..d15f00e26724 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -22,13 +22,13 @@ com.typesafe.akka akka-actor_${scala.binary.version} - 2.3.11 + 2.5.23 compile com.typesafe.akka akka-testkit_${scala.binary.version} - 2.3.11 + 2.5.23 test diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala index 36ad18bb8ef9..fb388d083282 100644 --- a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala +++ b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/rabit/RabitTracker.scala @@ -131,9 +131,7 @@ private[scala] class RabitTracker(numWorkers: Int, port: Option[Int] = None, } def stop(): Unit = { - if (!system.isTerminated) { - system.shutdown() - } + system.terminate() } /** @@ -170,12 +168,10 @@ private[scala] class RabitTracker(numWorkers: Int, port: Option[Int] = None, case Failure(e) => IRabitTracker.TrackerStatus.FAILURE.getStatusCode } - system.shutdown() + system.terminate() statusCode case Failure(ex: Throwable) => - if (!system.isTerminated) { - system.shutdown() - } + system.terminate() IRabitTracker.TrackerStatus.FAILURE.getStatusCode } } From fc9c772a33bf24862fcd2efe9756f5b3d3cc7810 Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Mon, 17 Jun 2019 12:22:58 -0400 Subject: [PATCH 02/11] put scala version in artifactId --- jvm-packages/README.md | 10 +++++----- jvm-packages/pom.xml | 2 +- jvm-packages/xgboost4j-example/pom.xml | 8 ++++---- jvm-packages/xgboost4j-flink/pom.xml | 6 +++--- jvm-packages/xgboost4j-spark/pom.xml | 6 +++--- jvm-packages/xgboost4j-tester/generate_pom.py | 14 +++++++------- jvm-packages/xgboost4j/pom.xml | 4 ++-- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/jvm-packages/README.md b/jvm-packages/README.md index 0f8cd16ea2c7..e03b166b3260 100644 --- a/jvm-packages/README.md +++ b/jvm-packages/README.md @@ -27,14 +27,14 @@ XGBoost4J, XGBoost4J-Spark, etc. in maven repository is compiled with g++-4.8.5 ``` ml.dmlc - xgboost4j + xgboost4j_2.12 latest_version_num ``` sbt ```sbt - "ml.dmlc" % "xgboost4j" % "latest_version_num" + "ml.dmlc" %% "xgboost4j" % "latest_version_num" ``` For the latest release version number, please check [here](https://github.com/dmlc/xgboost/releases). @@ -68,17 +68,17 @@ the add dependency as following: ``` ml.dmlc - xgboost4j + xgboost4j_2.12 latest_version_num ``` sbt ```sbt - "ml.dmlc" % "xgboost4j" % "latest_version_num" + "ml.dmlc" %% "xgboost4j" % "latest_version_num" ``` -For the latest release version number, please check [here](https://github.com/CodingCat/xgboost/tree/maven-repo/ml/dmlc/xgboost4j). +For the latest release version number, please check [here](https://github.com/CodingCat/xgboost/tree/maven-repo/ml/dmlc/xgboost4j_2.12). if you want to use `xgboost4j-spark`, you just need to replace xgboost4j with `xgboost4j-spark` diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml index 5401b9cbd084..a81520eac5d9 100644 --- a/jvm-packages/pom.xml +++ b/jvm-packages/pom.xml @@ -5,7 +5,7 @@ 4.0.0 ml.dmlc - xgboost-jvm + xgboost-jvm_2.12 0.90 pom XGBoost JVM Package diff --git a/jvm-packages/xgboost4j-example/pom.xml b/jvm-packages/xgboost4j-example/pom.xml index 0c628f9dde40..ac0d495d2523 100644 --- a/jvm-packages/xgboost4j-example/pom.xml +++ b/jvm-packages/xgboost4j-example/pom.xml @@ -5,10 +5,10 @@ 4.0.0 ml.dmlc - xgboost-jvm + xgboost-jvm_2.12 0.90 - xgboost4j-example + xgboost4j-example_2.12 0.90 jar @@ -25,7 +25,7 @@ ml.dmlc - xgboost4j-spark + xgboost4j-spark_${scala.binary.version} 0.90 @@ -36,7 +36,7 @@ ml.dmlc - xgboost4j-flink + xgboost4j-flink_${scala.binary.version} 0.90 diff --git a/jvm-packages/xgboost4j-flink/pom.xml b/jvm-packages/xgboost4j-flink/pom.xml index 3e249896e7d6..0be85d1ce7dd 100644 --- a/jvm-packages/xgboost4j-flink/pom.xml +++ b/jvm-packages/xgboost4j-flink/pom.xml @@ -5,10 +5,10 @@ 4.0.0 ml.dmlc - xgboost-jvm + xgboost-jvm_2.12 0.90 - xgboost4j-flink + xgboost4j-flink_2.12 0.90 @@ -25,7 +25,7 @@ ml.dmlc - xgboost4j + xgboost4j_${scala.binary.version} 0.90 diff --git a/jvm-packages/xgboost4j-spark/pom.xml b/jvm-packages/xgboost4j-spark/pom.xml index 2407a047fba8..09af42cf9950 100644 --- a/jvm-packages/xgboost4j-spark/pom.xml +++ b/jvm-packages/xgboost4j-spark/pom.xml @@ -5,10 +5,10 @@ 4.0.0 ml.dmlc - xgboost-jvm + xgboost-jvm_2.12 0.90 - xgboost4j-spark + xgboost4j-spark_2.12 @@ -23,7 +23,7 @@ ml.dmlc - xgboost4j + xgboost4j_${scala.binary.version} 0.90 diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py index 3f1099c61a24..0df208374903 100644 --- a/jvm-packages/xgboost4j-tester/generate_pom.py +++ b/jvm-packages/xgboost4j-tester/generate_pom.py @@ -8,10 +8,10 @@ 4.0.0 ml.dmlc - xgboost4j-tester + xgboost4j-tester_2.12 1.0-SNAPSHOT - xgboost4j-tester + xgboost4j-tester_2.12 UTF-8 @@ -97,12 +97,12 @@ ml.dmlc - xgboost4j + xgboost4j_${{scala.binary.version}} {xgboost4j_version} ml.dmlc - xgboost4j + xgboost4j_${{scala.binary.version}} {xgboost4j_version} tests test-jar @@ -110,12 +110,12 @@ ml.dmlc - xgboost4j-spark + xgboost4j-spark_${{scala.binary.version}} {xgboost4j_version} ml.dmlc - xgboost4j-example + xgboost4j-example_${{scala.binary.version}} {xgboost4j_version} @@ -186,7 +186,7 @@ 2.22.1 - ml.dmlc:xgboost4j + ml.dmlc:xgboost4j_2.12 diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml index d15f00e26724..af757e4396fb 100644 --- a/jvm-packages/xgboost4j/pom.xml +++ b/jvm-packages/xgboost4j/pom.xml @@ -5,10 +5,10 @@ 4.0.0 ml.dmlc - xgboost-jvm + xgboost-jvm_2.12 0.90 - xgboost4j + xgboost4j_2.12 0.90 jar From 73460a34d12f4f34088317e79aad4c6e320fd5f0 Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Mon, 17 Jun 2019 17:17:31 -0400 Subject: [PATCH 03/11] fix appveyor --- appveyor.yml | 2 +- jvm-packages/pom.xml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 4f14e239876d..fdf40cfba8b0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -98,7 +98,7 @@ build_script: cmake .. -G"%generator%" -DCMAKE_CONFIGURATION_TYPES="Release" -DR_LIB=ON && cmake --build . --target install --config Release ) - - if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j + - if /i "%target%" == "jvm" cd jvm-packages && mvn test -pl :xgboost4j_2.12 test_script: - cd %APPVEYOR_BUILD_FOLDER% diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml index a81520eac5d9..e74ca8bb0daf 100644 --- a/jvm-packages/pom.xml +++ b/jvm-packages/pom.xml @@ -375,5 +375,11 @@ 3.0.8 test + + org.scalactic + scalactic_${scala.binary.version} + 3.0.8 + test + From a95091935422915a329eabc62a8086914b31cbb4 Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Mon, 17 Jun 2019 22:26:58 -0400 Subject: [PATCH 04/11] fix for scaladoc issue that looks like https://github.com/scala/bug/issues/10509 --- .../src/main/java/ml/dmlc/xgboost4j/java/DMatrix.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/DMatrix.java b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/DMatrix.java index 4d8a49dba751..c62814717d2e 100644 --- a/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/DMatrix.java +++ b/jvm-packages/xgboost4j/src/main/java/ml/dmlc/xgboost4j/java/DMatrix.java @@ -78,7 +78,8 @@ public DMatrix(String dataPath) throws XGBoostError { * @throws XGBoostError */ @Deprecated - public DMatrix(long[] headers, int[] indices, float[] data, SparseType st) throws XGBoostError { + public DMatrix(long[] headers, int[] indices, float[] data, DMatrix.SparseType st) + throws XGBoostError { long[] out = new long[1]; if (st == SparseType.CSR) { XGBoostJNI.checkCall(XGBoostJNI.XGDMatrixCreateFromCSREx(headers, indices, data, 0, out)); @@ -100,7 +101,7 @@ public DMatrix(long[] headers, int[] indices, float[] data, SparseType st) throw * row number * @throws XGBoostError */ - public DMatrix(long[] headers, int[] indices, float[] data, SparseType st, int shapeParam) + public DMatrix(long[] headers, int[] indices, float[] data, DMatrix.SparseType st, int shapeParam) throws XGBoostError { long[] out = new long[1]; if (st == SparseType.CSR) { From 3e157d0b9d9c360236d6c35d49e6ddb2b6684ebd Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Tue, 18 Jun 2019 00:25:50 -0400 Subject: [PATCH 05/11] fix ci_build --- tests/ci_build/test_jvm_cross.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/ci_build/test_jvm_cross.sh b/tests/ci_build/test_jvm_cross.sh index 9a9f35011665..378846d651db 100755 --- a/tests/ci_build/test_jvm_cross.sh +++ b/tests/ci_build/test_jvm_cross.sh @@ -16,10 +16,10 @@ scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout) scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout) # Install XGBoost4J JAR into local Maven repository -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=jar -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark -Dversion=${xgboost4j_version} -Dpackaging=jar -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example -Dversion=${xgboost4j_version} -Dpackaging=jar +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar +mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar cd xgboost4j-tester # Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests @@ -28,14 +28,14 @@ python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_ mvn --no-transfer-progress package # Run integration tests with XGBoost4J -java -jar ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar +java -jar ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar # Run integration tests with XGBoost4J-Spark if [ ! -z "$RUN_INTEGRATION_TEST" ] then python3 get_iris.py - spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv - spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model + spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv + spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model fi set +x From 88bd1b596ffc24553ba87295626b27482e6c9415 Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Tue, 18 Jun 2019 10:41:35 -0400 Subject: [PATCH 06/11] update versions in generate_pom.py --- jvm-packages/xgboost4j-tester/generate_pom.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py index 0df208374903..fa59136c80f3 100644 --- a/jvm-packages/xgboost4j-tester/generate_pom.py +++ b/jvm-packages/xgboost4j-tester/generate_pom.py @@ -51,19 +51,25 @@ com.typesafe.akka akka-actor_${{scala.binary.version}} - 2.3.11 + 2.5.23 compile com.typesafe.akka akka-testkit_${{scala.binary.version}} - 2.3.11 + 2.5.23 test org.scalatest scalatest_${{scala.binary.version}} - 3.0.0 + 3.0.8 + test + + + org.scalactic + scalactic_${scala.binary.version} + 3.0.8 test From 8c5f5af1d59aaa28f36de54d628a6d6861e1afad Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Tue, 18 Jun 2019 11:34:20 -0400 Subject: [PATCH 07/11] fix generate_pom.py --- jvm-packages/xgboost4j-tester/generate_pom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py index fa59136c80f3..a1c19d53258d 100644 --- a/jvm-packages/xgboost4j-tester/generate_pom.py +++ b/jvm-packages/xgboost4j-tester/generate_pom.py @@ -68,7 +68,7 @@ org.scalactic - scalactic_${scala.binary.version} + scalactic_${{scala.binary.version}} 3.0.8 test From 21d96641e3afc94add70ccf02a24fec884d43cdd Mon Sep 17 00:00:00 2001 From: Koert Kuipers Date: Tue, 18 Jun 2019 14:20:26 -0400 Subject: [PATCH 08/11] apache does not have a download for spark 2.4.3 distro using scala 2.12 yet, so for now i use a tgz i put on s3 --- tests/ci_build/Dockerfile.jvm_cross | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross index aeb44ff82cd0..2e278158289d 100644 --- a/tests/ci_build/Dockerfile.jvm_cross +++ b/tests/ci_build/Dockerfile.jvm_cross @@ -20,9 +20,10 @@ RUN \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ ln -s /opt/apache-maven-3.6.1/ /opt/maven && \ # Spark - wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop2.7.tgz && \ - tar xvf spark-$SPARK_VERSION-bin-hadoop2.7.tgz -C /opt && \ - ln -s /opt/spark-$SPARK_VERSION-bin-hadoop2.7 /opt/spark + # This should be: wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz && \ + wget https://tresata-public.s3.amazonaws.com/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz && \ + tar xvf spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz -C /opt && \ + ln -s /opt/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7 /opt/spark ENV PATH=/opt/python/bin:/opt/spark/bin:/opt/maven/bin:$PATH From cd79cdb5952aa8bc6b6350e5e729f7b6db109cb6 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Mon, 15 Jul 2019 18:16:49 -0700 Subject: [PATCH 09/11] Upload spark-2.4.3-bin-scala2.12-hadoop2.7.tgz to our own S3 --- tests/ci_build/Dockerfile.jvm_cross | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross index 2e278158289d..53f7f0ae00ab 100644 --- a/tests/ci_build/Dockerfile.jvm_cross +++ b/tests/ci_build/Dockerfile.jvm_cross @@ -15,13 +15,14 @@ RUN \ # Python wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \ bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \ + pip install awscli && \ # Maven wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ ln -s /opt/apache-maven-3.6.1/ /opt/maven && \ # Spark - # This should be: wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz && \ - wget https://tresata-public.s3.amazonaws.com/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz && \ + # This should be: wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz + python -m awscli s3 cp s3:///xgboost-ci-jenkins-artifacts/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz . && \ tar xvf spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz -C /opt && \ ln -s /opt/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7 /opt/spark From 10d7f0c1e671f0c76aedd4821e87892ed1f0fb10 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Mon, 15 Jul 2019 18:18:52 -0700 Subject: [PATCH 10/11] Update Dockerfile.jvm_cross --- tests/ci_build/Dockerfile.jvm_cross | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross index 53f7f0ae00ab..9e611f109103 100644 --- a/tests/ci_build/Dockerfile.jvm_cross +++ b/tests/ci_build/Dockerfile.jvm_cross @@ -22,7 +22,7 @@ RUN \ ln -s /opt/apache-maven-3.6.1/ /opt/maven && \ # Spark # This should be: wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz - python -m awscli s3 cp s3:///xgboost-ci-jenkins-artifacts/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz . && \ + python -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz . && \ tar xvf spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz -C /opt && \ ln -s /opt/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7 /opt/spark From 654fb252bc5535201869e115b9d23c1a0882bdf6 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Mon, 15 Jul 2019 18:43:25 -0700 Subject: [PATCH 11/11] Update Dockerfile.jvm_cross --- tests/ci_build/Dockerfile.jvm_cross | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci_build/Dockerfile.jvm_cross b/tests/ci_build/Dockerfile.jvm_cross index 9e611f109103..db47fe33a2fa 100644 --- a/tests/ci_build/Dockerfile.jvm_cross +++ b/tests/ci_build/Dockerfile.jvm_cross @@ -15,14 +15,14 @@ RUN \ # Python wget https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \ bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /opt/python && \ - pip install awscli && \ + /opt/python/bin/pip install awscli && \ # Maven wget http://apache.osuosl.org/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz && \ tar xvf apache-maven-3.6.1-bin.tar.gz -C /opt && \ ln -s /opt/apache-maven-3.6.1/ /opt/maven && \ # Spark # This should be: wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz - python -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz . && \ + /opt/python/bin/python -m awscli s3 cp s3://xgboost-ci-jenkins-artifacts/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz . && \ tar xvf spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7.tgz -C /opt && \ ln -s /opt/spark-$SPARK_VERSION-bin-scala2.12-hadoop2.7 /opt/spark