diff --git a/.travis.yml b/.travis.yml index ac80e8a6959..cc8fbb64b35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,35 +22,34 @@ before_install: - "sh -e /etc/init.d/xvfb start" install: - - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B + - mvn package -DskipTests -Phadoop-2.4 -Ppyspark -B before_script: - - + - ./testing/startYarnCluster.sh 2.5.2 script: # spark 1.4 - - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/startSparkCluster.sh 1.4.0 2.3 - - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/stopSparkCluster.sh 1.4.0 2.3 + - mvn package -Pbuild-distr -Pyarn -Phadoop-2.4 -Ppyspark -B + - ./testing/startSparkCluster.sh 1.4.1 2.4 + - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B + - ./testing/stopSparkCluster.sh 1.4.1 2.4 + - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B + # spark 1.3 - - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B - - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - ./testing/stopSparkCluster.sh 1.3.1 2.3 + - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' + - ./testing/startSparkCluster.sh 1.3.1 2.4 + - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - ./testing/stopSparkCluster.sh 1.3.1 2.4 # spark 1.2 - - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - ./testing/stopSparkCluster.sh 1.2.1 2.3 + - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' + - ./testing/startSparkCluster.sh 1.2.1 2.4 + - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - ./testing/stopSparkCluster.sh 1.2.1 2.4 # spark 1.1 - - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - ./testing/stopSparkCluster.sh 1.1.1 2.3 + - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' + - ./testing/startSparkCluster.sh 1.1.1 2.4 + - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - ./testing/stopSparkCluster.sh 1.1.1 2.4 after_failure: - cat target/rat.txt @@ -59,7 +58,7 @@ after_failure: - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out after_script: - - + - ./testing/stopYarnCluster.sh 2.5.2 notifications: slack: diff --git a/pom.xml b/pom.xml index 9e5f54e7002..db806aeba3a 100644 --- a/pom.xml +++ b/pom.xml @@ -431,6 +431,7 @@ Roadmap.md conf/interpreter.json spark-*-bin*/** + hadoop-*/** diff --git a/testing/startYarnCluster.sh b/testing/startYarnCluster.sh new file mode 100755 index 00000000000..1ee8c5ab7ed --- /dev/null +++ b/testing/startYarnCluster.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +if [ $# -ne 1 ]; then + echo "usage) $0 [hadoop version]" + echo " eg) $0 2.6.0" + exit 1 +fi + + + +HADOOP_VERSION="${1}" + +FWDIR=$(dirname "${BASH_SOURCE-$0}") +ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" +export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION} + +if [ ! -d "${HADOOP_HOME}" ]; then + wget -q http://www.us.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + tar zxf hadoop-${HADOOP_VERSION}.tar.gz +fi + +DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# start resourceManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager + +# start nodeManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start nodemanager diff --git a/testing/stopYarnCluster.sh b/testing/stopYarnCluster.sh new file mode 100755 index 00000000000..7a64fd6f180 --- /dev/null +++ b/testing/stopYarnCluster.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ $# -ne 1 ]; then + echo "usage) $0 [hadoop version]" + echo " eg) $0 2.6.0" + exit 1 +fi + +HADOOP_VERSION="${1}" + +FWDIR=$(dirname "${BASH_SOURCE-$0}") +ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" +export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION} + +DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# stop nodeManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop nodemanager + +# stop resourceManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 393dc7bcabc..4c312efa24d 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -112,7 +112,11 @@ protected static void startUp() throws Exception { InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0); // set spark master - sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + if ("true".equals(System.getenv("YARN"))) { + sparkIntpSetting.getProperties().setProperty("master", "yarn-client"); + } else { + sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + } // set spark home for pyspark sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome()); @@ -123,6 +127,13 @@ protected static void startUp() throws Exception { // assume first one is spark InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0); + // set spark master + if ("true".equals(System.getenv("YARN"))) { + sparkIntpSetting.getProperties().setProperty("master", "yarn-client"); + } else { + sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + } + String sparkHome = getSparkHome(); if (sparkHome != null) { // set spark home for pyspark @@ -145,9 +156,13 @@ private static String getHostname() { } private static String getSparkHome() { - String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir"))); - System.out.println("SPARK HOME detected " + sparkHome); - return sparkHome; + if (System.getenv("SPARK_HOME") != null) { + return new File(System.getenv("SPARK_HOME")).getAbsolutePath(); + } else { // autodetect spark.home that ./testing/startSparkCluster.sh creates + String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir"))); + System.out.println("SPARK HOME detected " + sparkHome); + return sparkHome; + } } boolean isPyspark() {