diff --git a/.travis.yml b/.travis.yml
index ac80e8a6959..cc8fbb64b35 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,35 +22,34 @@ before_install:
- "sh -e /etc/init.d/xvfb start"
install:
- - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B
+ - mvn package -DskipTests -Phadoop-2.4 -Ppyspark -B
before_script:
- -
+ - ./testing/startYarnCluster.sh 2.5.2
script:
# spark 1.4
- - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B
- - ./testing/startSparkCluster.sh 1.4.0 2.3
- - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B
- - ./testing/stopSparkCluster.sh 1.4.0 2.3
+ - mvn package -Pbuild-distr -Pyarn -Phadoop-2.4 -Ppyspark -B
+ - ./testing/startSparkCluster.sh 1.4.1 2.4
+ - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B
+ - ./testing/stopSparkCluster.sh 1.4.1 2.4
+ - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B
+
# spark 1.3
- - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B
- - ./testing/startSparkCluster.sh 1.3.1 2.3
- - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- - ./testing/stopSparkCluster.sh 1.3.1 2.3
+ - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark'
+ - ./testing/startSparkCluster.sh 1.3.1 2.4
+ - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
+ - ./testing/stopSparkCluster.sh 1.3.1 2.4
# spark 1.2
- - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- - mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B
- - ./testing/startSparkCluster.sh 1.2.1 2.3
- - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- - ./testing/stopSparkCluster.sh 1.2.1 2.3
+ - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark'
+ - ./testing/startSparkCluster.sh 1.2.1 2.4
+ - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
+ - ./testing/stopSparkCluster.sh 1.2.1 2.4
# spark 1.1
- - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B
- - ./testing/startSparkCluster.sh 1.1.1 2.3
- - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- - ./testing/stopSparkCluster.sh 1.1.1 2.3
+ - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark'
+ - ./testing/startSparkCluster.sh 1.1.1 2.4
+ - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server'
+ - ./testing/stopSparkCluster.sh 1.1.1 2.4
after_failure:
- cat target/rat.txt
@@ -59,7 +58,7 @@ after_failure:
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
after_script:
- -
+ - ./testing/stopYarnCluster.sh 2.5.2
notifications:
slack:
diff --git a/pom.xml b/pom.xml
index 9e5f54e7002..db806aeba3a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -431,6 +431,7 @@
Roadmap.md
conf/interpreter.json
spark-*-bin*/**
+ hadoop-*/**
diff --git a/testing/startYarnCluster.sh b/testing/startYarnCluster.sh
new file mode 100755
index 00000000000..1ee8c5ab7ed
--- /dev/null
+++ b/testing/startYarnCluster.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+if [ $# -ne 1 ]; then
+ echo "usage) $0 [hadoop version]"
+ echo " eg) $0 2.6.0"
+ exit 1
+fi
+
+
+
+HADOOP_VERSION="${1}"
+
+FWDIR=$(dirname "${BASH_SOURCE-$0}")
+ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
+export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION}
+
+if [ ! -d "${HADOOP_HOME}" ]; then
+ wget -q http://www.us.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
+ tar zxf hadoop-${HADOOP_VERSION}.tar.gz
+fi
+
+DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
+. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+
+# start resourceManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager
+
+# start nodeManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start nodemanager
diff --git a/testing/stopYarnCluster.sh b/testing/stopYarnCluster.sh
new file mode 100755
index 00000000000..7a64fd6f180
--- /dev/null
+++ b/testing/stopYarnCluster.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ $# -ne 1 ]; then
+ echo "usage) $0 [hadoop version]"
+ echo " eg) $0 2.6.0"
+ exit 1
+fi
+
+HADOOP_VERSION="${1}"
+
+FWDIR=$(dirname "${BASH_SOURCE-$0}")
+ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
+export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION}
+
+DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
+. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+
+# stop nodeManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop nodemanager
+
+# stop resourceManager
+$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index 393dc7bcabc..4c312efa24d 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -112,7 +112,11 @@ protected static void startUp() throws Exception {
InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0);
// set spark master
- sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+ if ("true".equals(System.getenv("YARN"))) {
+ sparkIntpSetting.getProperties().setProperty("master", "yarn-client");
+ } else {
+ sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+ }
// set spark home for pyspark
sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
@@ -123,6 +127,13 @@ protected static void startUp() throws Exception {
// assume first one is spark
InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0);
+ // set spark master
+ if ("true".equals(System.getenv("YARN"))) {
+ sparkIntpSetting.getProperties().setProperty("master", "yarn-client");
+ } else {
+ sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+ }
+
String sparkHome = getSparkHome();
if (sparkHome != null) {
// set spark home for pyspark
@@ -145,9 +156,13 @@ private static String getHostname() {
}
private static String getSparkHome() {
- String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir")));
- System.out.println("SPARK HOME detected " + sparkHome);
- return sparkHome;
+ if (System.getenv("SPARK_HOME") != null) {
+ return new File(System.getenv("SPARK_HOME")).getAbsolutePath();
+ } else { // autodetect spark.home that ./testing/startSparkCluster.sh creates
+ String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir")));
+ System.out.println("SPARK HOME detected " + sparkHome);
+ return sparkHome;
+ }
}
boolean isPyspark() {