From 5615eb9550e5752df1d6c8bf31b2e50fd0a563cf Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 10:03:45 -0800 Subject: [PATCH 1/8] use build matrix --- .travis.yml | 57 ++++++++++++----------------------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74f180597a0..93fa8f1968a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,61 +14,30 @@ # limitations under the License. language: java -jdk: - - oraclejdk7 + +matrix: + include: + - jdk: "oraclejdk7" + env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" + - jdk: "oraclejdk7" + env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -Pscalding" before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" install: - - mvn package -DskipTests -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B + - mvn package -DskipTests ${PROFILE} -B before_script: - script: - # spark 1.6 - - mvn package -Pbuild-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B - - ./testing/startSparkCluster.sh 1.6.0 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.6.0-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn verify -Pusing-packaged-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B - - ./testing/stopSparkCluster.sh 1.6.0 2.3 - # spark 1.5 - - rm -rf `pwd`/interpreter/spark - - mvn package -DskipTests -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.5.2 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.5.2-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.5 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.5.2 2.3 - # spark 1.4 - - rm -rf `pwd`/interpreter/spark - - mvn package -DskipTests -Pspark-1.4 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.4.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.4 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.4.1 2.3 - # spark 1.3 - - rm -rf `pwd`/interpreter/spark - - mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.3.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.3.1 2.3 - # spark 1.2 - - rm -rf `pwd`/interpreter/spark - - mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.2.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.2.1 2.3 -# spark 1.1 - - rm -rf `pwd`/interpreter/spark - - mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark' - - ./testing/startSparkCluster.sh 1.1.1 2.3 - - echo "export SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3" > conf/zeppelin-env.sh - - mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false - - ./testing/stopSparkCluster.sh 1.1.1 2.3 + - mvn package -Pbuild-distr ${PROFILE} -B + - ./testing/startSparkCluster.sh ${SPARK_VER} ${HADOOP_VER} + - echo "export SPARK_HOME=`pwd`/spark-${SPARK_VER}-bin-hadoop${HADOOP_VER}" > conf/zeppelin-env.sh + - mvn verify -Pusing-packaged-distr ${PROFILE} -B + - ./testing/stopSparkCluster.sh ${SPARK_VER} ${HADOOP_VER} after_failure: - cat target/rat.txt From 69532784e3c70d2f018969f308ed2d25e4830dc3 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 10:19:01 -0800 Subject: [PATCH 2/8] Update --- .travis.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93fa8f1968a..d102e0f9f51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,27 +17,26 @@ language: java matrix: include: + # Test all modules - jdk: "oraclejdk7" - env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" + env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" + # Test only spark modules - jdk: "oraclejdk7" - env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -Pscalding" + env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -pl 'zeppelin-interpreter,spark-dependencies,spark'" BUILD_FLAG="package -DskipTests" TEST_FLAG="package" before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" install: - - mvn package -DskipTests ${PROFILE} -B + - mvn $BUILD_FLAG $PROFILE -B before_script: - - + - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER + - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh script: - - mvn package -Pbuild-distr ${PROFILE} -B - - ./testing/startSparkCluster.sh ${SPARK_VER} ${HADOOP_VER} - - echo "export SPARK_HOME=`pwd`/spark-${SPARK_VER}-bin-hadoop${HADOOP_VER}" > conf/zeppelin-env.sh - - mvn verify -Pusing-packaged-distr ${PROFILE} -B - - ./testing/stopSparkCluster.sh ${SPARK_VER} ${HADOOP_VER} + - mvn $TEST_FLAG $PROFILE -B after_failure: - cat target/rat.txt @@ -46,7 +45,7 @@ after_failure: - cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out after_script: - - + - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER notifications: slack: From 3e523c547cc57ee8bc855e3ce821f3de1cf9e917 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 10:48:09 -0800 Subject: [PATCH 3/8] update spark module test --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d102e0f9f51..555fbbce1a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,10 +19,15 @@ matrix: include: # Test all modules - jdk: "oraclejdk7" + env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" + + # Test all modules with jdk8 + - jdk: "oraclejdk8" env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" - # Test only spark modules + + # Test spark module for 1.5.2 - jdk: "oraclejdk7" - env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -pl 'zeppelin-interpreter,spark-dependencies,spark'" BUILD_FLAG="package -DskipTests" TEST_FLAG="package" + env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-web,spark-dependencies,spark'" BUILD_FLAG="package -DskipTests" TEST_FLAG="package" before_install: - "export DISPLAY=:99.0" From f8f69ae9bcd5f59b42b5914514584217fcab1c32 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 11:03:17 -0800 Subject: [PATCH 4/8] try in a different way --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 555fbbce1a0..ec3960e953c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ matrix: # Test spark module for 1.5.2 - jdk: "oraclejdk7" - env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-web,spark-dependencies,spark'" BUILD_FLAG="package -DskipTests" TEST_FLAG="package" + env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify -pl 'zeppelin-interpreter,spark-dependencies,spark'" before_install: - "export DISPLAY=:99.0" From 8e0e94f975536c7501564afa51dd86ca7817422f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 11:14:40 -0800 Subject: [PATCH 5/8] Another try --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ec3960e953c..e516ffb6104 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ matrix: # Test spark module for 1.5.2 - jdk: "oraclejdk7" - env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify -pl 'zeppelin-interpreter,spark-dependencies,spark'" + env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" before_install: - "export DISPLAY=:99.0" From a1ce0d76f2c79f232069fdfa0060d72bb082e79b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 14:09:30 -0800 Subject: [PATCH 6/8] Enable test for other spark versions --- .travis.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.travis.yml b/.travis.yml index e516ffb6104..4f9ea5210b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,22 @@ matrix: - jdk: "oraclejdk7" env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" + # Test spark module for 1.4.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" + + # Test spark module for 1.3.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" + + # Test spark module for 1.2.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.2.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" + + # Test spark module for 1.1.1 + - jdk: "oraclejdk7" + env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" + before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" From 193dcd26e02b8ed8e0ec221356c3e6d0bddd7a33 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 14:19:27 -0800 Subject: [PATCH 7/8] Add scalding profile --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f9ea5210b8..5f0f0f452ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ matrix: include: # Test all modules - jdk: "oraclejdk7" - env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" + env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" # Test all modules with jdk8 - jdk: "oraclejdk8" From c13086a7520cb10ddaede9d725671fece1fdcee8 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Jan 2016 15:43:54 -0800 Subject: [PATCH 8/8] Remove jdk8 --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f0f0f452ce..2619ed1c7cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,10 +21,6 @@ matrix: - jdk: "oraclejdk7" env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" - # Test all modules with jdk8 - - jdk: "oraclejdk8" - env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" - # Test spark module for 1.5.2 - jdk: "oraclejdk7" env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"