From 63fc1eebd2f4809203fa6fbf015b4e2160784423 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 12:36:28 -0700 Subject: [PATCH 01/10] test ZeppelinServerClusterTest --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac80e8a6959..3222d173abc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,19 +37,19 @@ script: - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' + - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From 6fe43995cea574753016be3f491457d28e0f8f3e Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 12:40:49 -0700 Subject: [PATCH 02/10] Add script for start/stop yarn cluster --- pom.xml | 1 + testing/startYarnCluster.sh | 47 +++++++++++++++++++++++++++++++++++++ testing/stopYarnCluster.sh | 39 ++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100755 testing/startYarnCluster.sh create mode 100755 testing/stopYarnCluster.sh diff --git a/pom.xml b/pom.xml index 9e5f54e7002..db806aeba3a 100644 --- a/pom.xml +++ b/pom.xml @@ -431,6 +431,7 @@ Roadmap.md conf/interpreter.json spark-*-bin*/** + hadoop-*/** diff --git a/testing/startYarnCluster.sh b/testing/startYarnCluster.sh new file mode 100755 index 00000000000..1ee8c5ab7ed --- /dev/null +++ b/testing/startYarnCluster.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +if [ $# -ne 1 ]; then + echo "usage) $0 [hadoop version]" + echo " eg) $0 2.6.0" + exit 1 +fi + + + +HADOOP_VERSION="${1}" + +FWDIR=$(dirname "${BASH_SOURCE-$0}") +ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" +export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION} + +if [ ! -d "${HADOOP_HOME}" ]; then + wget -q http://www.us.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz + tar zxf hadoop-${HADOOP_VERSION}.tar.gz +fi + +DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# start resourceManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start resourcemanager + +# start nodeManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR start nodemanager diff --git a/testing/stopYarnCluster.sh b/testing/stopYarnCluster.sh new file mode 100755 index 00000000000..7a64fd6f180 --- /dev/null +++ b/testing/stopYarnCluster.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ $# -ne 1 ]; then + echo "usage) $0 [hadoop version]" + echo " eg) $0 2.6.0" + exit 1 +fi + +HADOOP_VERSION="${1}" + +FWDIR=$(dirname "${BASH_SOURCE-$0}") +ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" +export HADOOP_HOME=${ZEPPELIN_HOME}/hadoop-${HADOOP_VERSION} + +DEFAULT_LIBEXEC_DIR=${HADOOP_HOME}/libexec +HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} +. $HADOOP_LIBEXEC_DIR/yarn-config.sh + +# stop nodeManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop nodemanager + +# stop resourceManager +$HADOOP_HOME/sbin/yarn-daemon.sh --config $YARN_CONF_DIR stop resourcemanager From 57ae6e31d54c755f884a6728074706b72efa5f82 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 13:30:22 -0700 Subject: [PATCH 03/10] verification with distribution package would be enough for latest one --- .travis.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3222d173abc..39c5e6cd65d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,21 +35,18 @@ script: - ./testing/stopSparkCluster.sh 1.4.0 2.3 # spark 1.3 - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.3.1 2.3 # spark 1.2 - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.2.1 2.3 # spark 1.1 - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.1.1 2.3 after_failure: From 17d0e16be9a65b6289d610881442ce6070121e00 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 13:42:26 -0700 Subject: [PATCH 04/10] hadoop 2.3 -> 2.4 --- .travis.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 39c5e6cd65d..262696ebb96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,32 +22,32 @@ before_install: - "sh -e /etc/init.d/xvfb start" install: - - mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B + - mvn package -DskipTests -Phadoop-2.4 -Ppyspark -B before_script: - script: # spark 1.4 - - mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/startSparkCluster.sh 1.4.0 2.3 - - SPARK_HOME=`pwd`/spark-1.4.0-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B - - ./testing/stopSparkCluster.sh 1.4.0 2.3 + - mvn package -Pbuild-distr -Phadoop-2.4 -Ppyspark -B + - ./testing/startSparkCluster.sh 1.4.0 2.4 + - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Phadoop-2.4 -Ppyspark -B + - ./testing/stopSparkCluster.sh 1.4.0 2.4 # spark 1.3 - - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - ./testing/startSparkCluster.sh 1.3.1 2.3 - - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - - ./testing/stopSparkCluster.sh 1.3.1 2.3 + - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' + - ./testing/startSparkCluster.sh 1.3.1 2.4 + - SPARK_HOME=./spark-1.3.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - ./testing/stopSparkCluster.sh 1.3.1 2.4 # spark 1.2 - - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - ./testing/startSparkCluster.sh 1.2.1 2.3 - - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - - ./testing/stopSparkCluster.sh 1.2.1 2.3 + - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' + - ./testing/startSparkCluster.sh 1.2.1 2.4 + - SPARK_HOME=./spark-1.2.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - ./testing/stopSparkCluster.sh 1.2.1 2.4 # spark 1.1 - - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark' - - ./testing/startSparkCluster.sh 1.1.1 2.3 - - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - - ./testing/stopSparkCluster.sh 1.1.1 2.3 + - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' + - ./testing/startSparkCluster.sh 1.1.1 2.4 + - SPARK_HOME=./spark-1.1.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - ./testing/stopSparkCluster.sh 1.1.1 2.4 after_failure: - cat target/rat.txt From 2594e95add5b72f109ab6a23ff6b8b37d256e874 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 13:45:06 -0700 Subject: [PATCH 05/10] Test with Yarn --- .travis.yml | 4 ++++ .../apache/zeppelin/rest/AbstractTestRestApi.java | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 262696ebb96..0685311f755 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,10 @@ script: - ./testing/startSparkCluster.sh 1.4.0 2.4 - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Phadoop-2.4 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.4 + - ./testing/startYarnCluster.sh 2.5.2 + - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Phadoop-2.4 -B + - ./testing/stopYarnCluster.sh 2.5.2 + # spark 1.3 - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' - ./testing/startSparkCluster.sh 1.3.1 2.4 diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 393dc7bcabc..d71e8cdaead 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -112,7 +112,11 @@ protected static void startUp() throws Exception { InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0); // set spark master - sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + if ("true".equals(System.getenv("YARN"))) { + sparkIntpSetting.getProperties().setProperty("master", "yarn-client"); + } else { + sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + } // set spark home for pyspark sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome()); @@ -123,6 +127,13 @@ protected static void startUp() throws Exception { // assume first one is spark InterpreterSetting sparkIntpSetting = ZeppelinServer.notebook.getInterpreterFactory().get().get(0); + // set spark master + if ("true".equals(System.getenv("YARN"))) { + sparkIntpSetting.getProperties().setProperty("master", "yarn-client"); + } else { + sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + } + String sparkHome = getSparkHome(); if (sparkHome != null) { // set spark home for pyspark From d20a1a6d745aec57d645cf1d9c7cd90f9dbb2836 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 14:26:31 -0700 Subject: [PATCH 06/10] User SPARK_HOME env variable for spark.home when it is defined --- .../org/apache/zeppelin/rest/AbstractTestRestApi.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index d71e8cdaead..4c312efa24d 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -156,9 +156,13 @@ private static String getHostname() { } private static String getSparkHome() { - String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir"))); - System.out.println("SPARK HOME detected " + sparkHome); - return sparkHome; + if (System.getenv("SPARK_HOME") != null) { + return new File(System.getenv("SPARK_HOME")).getAbsolutePath(); + } else { // autodetect spark.home that ./testing/startSparkCluster.sh creates + String sparkHome = getSparkHomeRecursively(new File(System.getProperty("user.dir"))); + System.out.println("SPARK HOME detected " + sparkHome); + return sparkHome; + } } boolean isPyspark() { From 585176381a9c5bf685fca4f7b1c13670e3657fb6 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 15:06:30 -0700 Subject: [PATCH 07/10] Add yarn profile for ci test --- .travis.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0685311f755..51865469705 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,17 +25,16 @@ install: - mvn package -DskipTests -Phadoop-2.4 -Ppyspark -B before_script: - - + - ./testing/startYarnCluster.sh 2.5.2 script: # spark 1.4 - - mvn package -Pbuild-distr -Phadoop-2.4 -Ppyspark -B + - mvn package -Pbuild-distr -Pyarn -Phadoop-2.6 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.0 2.4 - - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Phadoop-2.4 -Ppyspark -B + - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -Ppyspark -B - ./testing/stopSparkCluster.sh 1.4.0 2.4 - - ./testing/startYarnCluster.sh 2.5.2 - - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Phadoop-2.4 -B - - ./testing/stopYarnCluster.sh 2.5.2 + - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B + # spark 1.3 - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' @@ -60,7 +59,7 @@ after_failure: - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out after_script: - - + - ./testing/stopYarnCluster.sh 2.5.2 notifications: slack: From 7548c73fdf9940766cf066753c17575789e13002 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 27 Jun 2015 18:19:55 -0700 Subject: [PATCH 08/10] Try absolute path for SPARK_HOME --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51865469705..189013719db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,9 +31,9 @@ script: # spark 1.4 - mvn package -Pbuild-distr -Pyarn -Phadoop-2.6 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.0 2.4 - - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -Ppyspark -B + - SPARK_HOME=$TRAVIS_BUILD_DIR/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B - ./testing/stopSparkCluster.sh 1.4.0 2.4 - - SPARK_HOME=./spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B + - SPARK_HOME=$TRAVIS_BUILD_DIR/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B # spark 1.3 From 92247e2e3ed3ff9814a5fdd24194addfa31f8c02 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 5 Aug 2015 14:33:48 -0700 Subject: [PATCH 09/10] spark 1.4.0 -> 1.4.1 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 189013719db..47553c2a506 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,10 +29,10 @@ before_script: script: # spark 1.4 - - mvn package -Pbuild-distr -Pyarn -Phadoop-2.6 -Ppyspark -B - - ./testing/startSparkCluster.sh 1.4.0 2.4 + - mvn package -Pbuild-distr -Pyarn -Phadoop-2.4 -Ppyspark -B + - ./testing/startSparkCluster.sh 1.4.1 2.4 - SPARK_HOME=$TRAVIS_BUILD_DIR/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B - - ./testing/stopSparkCluster.sh 1.4.0 2.4 + - ./testing/stopSparkCluster.sh 1.4.1 2.4 - SPARK_HOME=$TRAVIS_BUILD_DIR/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B From 42dd3dbfce185f079f32ec8c80ddb0bc7c8b428b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 5 Aug 2015 16:48:58 -0700 Subject: [PATCH 10/10] Set SPARK_HOME absolute path --- .travis.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47553c2a506..cc8fbb64b35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,25 +31,24 @@ script: # spark 1.4 - mvn package -Pbuild-distr -Pyarn -Phadoop-2.4 -Ppyspark -B - ./testing/startSparkCluster.sh 1.4.1 2.4 - - SPARK_HOME=$TRAVIS_BUILD_DIR/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B + - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B - ./testing/stopSparkCluster.sh 1.4.1 2.4 - - SPARK_HOME=$TRAVIS_BUILD_DIR/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B - + - SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.5 HADOOP_CONF_DIR=./hadoop-2.5.2/etc/hadoop YARN=true mvn verify -Pusing-packaged-distr -Pyarn -Phadoop-2.4 -B # spark 1.3 - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' - ./testing/startSparkCluster.sh 1.3.1 2.4 - - SPARK_HOME=./spark-1.3.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.3.1 2.4 # spark 1.2 - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' - ./testing/startSparkCluster.sh 1.2.1 2.4 - - SPARK_HOME=./spark-1.2.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.2.1 2.4 # spark 1.1 - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.4 -B -pl 'zeppelin-interpreter,spark' - ./testing/startSparkCluster.sh 1.1.1 2.4 - - SPARK_HOME=./spark-1.1.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' + - SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.4 mvn verify -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' - ./testing/stopSparkCluster.sh 1.1.1 2.4 after_failure: