Enable CI against Kubernetes 1.7 (#37)

* adding e2e-prow script

* uncomment lines

* newline

* newline

* Address one round of comments

* Update e2e-prow.sh

Author: foxish

dev/dev-run-integration-tests.sh

@@ -17,6 +17,8 @@
 # limitations under the License.
 #
 
+source ./include/util.sh
+
 TEST_ROOT_DIR=$(git rev-parse --show-toplevel)
 BRANCH="master"
 SPARK_REPO="https://github.com/apache/spark"
@@ -78,21 +80,7 @@ done
 if [[ $SPARK_TGZ == "N/A" ]];
 then
   echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH."
-
-  # clone spark distribution if needed.
-  if [ -d "$SPARK_REPO_LOCAL_DIR" ];
-  then
-    (cd $SPARK_REPO_LOCAL_DIR && git fetch origin $BRANCH);
-  else
-    mkdir -p $SPARK_REPO_LOCAL_DIR;
-    git clone -b $BRANCH --single-branch $SPARK_REPO $SPARK_REPO_LOCAL_DIR;
-  fi
-  cd $SPARK_REPO_LOCAL_DIR
-  git checkout -B $BRANCH origin/$BRANCH
-  ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests;
-  SPARK_TGZ=$(find $SPARK_REPO_LOCAL_DIR -name spark-*.tgz)
-  echo "Built Spark TGZ at $SPARK_TGZ".
-  cd -
+  clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH
 fi
 
 cd $TEST_ROOT_DIR

e2e/e2e-prow.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### This script is used by Kubernetes Test Infrastructure to run integration tests.
+### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow
+
+set -ex
+
+# Include requisite scripts
+source ./include/util.sh
+
+TEST_ROOT_DIR=$(git rev-parse --show-toplevel)
+BRANCH="master"
+SPARK_REPO="https://github.com/apache/spark"
+SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark"
+
+## Install basic dependencies
+## These are for the kubekins-e2e environment in https://github.com/kubernetes/test-infra/tree/master/images/kubekins-e2e
+echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list
+apt-get update && apt-get install -y curl wget git tar uuid-runtime
+apt-get install -t jessie-backports -y openjdk-8-jdk
+
+# Set up config.
+MASTER=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?")
+
+# Special GCP project for publishing docker images built by test.
+IMAGE_REPO="gcr.io/spark-testing-191023"
+
+# Cloning the spark distribution
+echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH."
+clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH
+
+# Spark distribution
+properties=(
+  -Dspark.kubernetes.test.master=k8s://$MASTER \
+  -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \
+  -Dspark.kubernetes.test.sparkTgz="$SPARK_TGZ" \
+  -Dspark.kubernetes.test.deployMode=cloud \
+  -Dspark.kubernetes.test.namespace=default \
+  -Dspark.kubernetes.test.serviceAccountName=default
+)
+
+# Run tests.
+echo "Starting test with ${properties[@]}"
+build/mvn integration-test "${properties[@]}"
+
+# Copy out the junit xml files for consumption by k8s test-infra.
+ls -1 ./integration-test/target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done

include/util.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+clone_build_spark() {
+  spark_repo=$1
+  spark_repo_local_dir=$2
+  branch=$3
+  pushd .
+
+  # clone spark distribution if needed.
+  if [ -d "$spark_repo_local_dir" ];
+  then
+    (cd $spark_repo_local_dir && git fetch origin $branch);
+  else
+    mkdir -p $spark_repo_local_dir;
+    git clone -b $branch --single-branch $spark_repo $spark_repo_local_dir;
+  fi
+  cd $spark_repo_local_dir
+  git checkout -B $branch origin/$branch
+  ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests;
+  SPARK_TGZ=$(find $spark_repo_local_dir -name spark-*.tgz)
+
+  popd
+}