From 8072b13d037cc60603d7dcdb83f61150f46332a6 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Mon, 6 Dec 2021 17:33:46 +0900 Subject: [PATCH 1/3] Change dev-run-integration-tests.sh to allow to specify a custom Dockerfile. --- .../kubernetes/integration-tests/README.md | 14 ++++++++++++++ .../dev/dev-run-integration-tests.sh | 12 +++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 9a0689f7c451..9cdc9c3d70ff 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -17,6 +17,13 @@ To run tests with Java 11 instead of Java 8, use `--java-image-tag` to specify t ./dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim +To run tests with a custom docker image, use `--docker-file` to specify the Dockerfile. +Note that if both `--docker-file` and `--java-image-tag` are used, `--docker-file` is preferred, +and the custom Dockerfile need to include a Java installation by itself. +Dockerfile.java17 is an example of custom Dockerfile, and you can specify it to run tests with Java 17. + + ./dev/dev-run-integration-tests.sh --docker-file ../docker/src/main/dockerfiles/spark/Dockerfile.java17 + To run tests with Hadoop 2.7 instead of Hadoop 3.2, use `--hadoop-profile`. ./dev/dev-run-integration-tests.sh --hadoop-profile hadoop-2.7 @@ -237,6 +244,13 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro spark-r + + spark.kubernetes.test.dockerFile + + The path to the custom Dockerfile + + N/A + spark.kubernetes.test.namespace diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 8d10985b4f26..1b67d95fa653 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -28,6 +28,7 @@ BASE_IMAGE_NAME= JVM_IMAGE_NAME= PYTHON_IMAGE_NAME= R_IMAGE_NAME= +DOCKER_FILE= SPARK_MASTER= NAMESPACE= SERVICE_ACCOUNT= @@ -70,6 +71,10 @@ while (( "$#" )); do SPARK_TGZ="$2" shift ;; + --docker-file) + DOCKER_FILE="$2" + shift + ;; --spark-master) SPARK_MASTER="$2" shift @@ -143,6 +148,11 @@ then properties=( ${properties[@]} -Dspark.kubernetes.test.javaImageTag=$JAVA_IMAGE_TAG ) fi +if [ -n "$DOCKER_FILE" ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.dockerFile=$(realpath $DOCKER_FILE) ) +fi + if [ -n "$NAMESPACE" ]; then properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE ) @@ -180,4 +190,4 @@ properties+=( -Dlog4j.logger.org.apache.spark=DEBUG ) -$TEST_ROOT_DIR/build/mvn install -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests $BUILD_DEPENDENCIES_MVN_FLAG -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]} +(cd $TEST_ROOT_DIR; ./build/mvn install -pl resource-managers/kubernetes/integration-tests $BUILD_DEPENDENCIES_MVN_FLAG -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}) From 26eca4182c3a8f1e26842beba6880bda6704356e Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 7 Dec 2021 02:34:33 +0900 Subject: [PATCH 2/3] Use Python's os.path.realpath instead of realpath command. --- .../dev/dev-run-integration-tests.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 1b67d95fa653..b3f87088ef5e 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -150,7 +150,10 @@ fi if [ -n "$DOCKER_FILE" ]; then - properties=( ${properties[@]} -Dspark.kubernetes.test.dockerFile=$(realpath $DOCKER_FILE) ) + properties=( + ${properties[@]} + -Dspark.kubernetes.test.dockerFile=$( + python3 -c "import os.path; print(os.path.realpath(\"$DOCKER_FILE\"))") ) fi if [ -n "$NAMESPACE" ]; @@ -190,4 +193,14 @@ properties+=( -Dlog4j.logger.org.apache.spark=DEBUG ) -(cd $TEST_ROOT_DIR; ./build/mvn install -pl resource-managers/kubernetes/integration-tests $BUILD_DEPENDENCIES_MVN_FLAG -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}) +( + cd $TEST_ROOT_DIR; + ./build/mvn install \ + -pl resource-managers/kubernetes/integration-tests \ + $BUILD_DEPENDENCIES_MVN_FLAG \ + -Pscala-$SCALA_VERSION \ + -P$HADOOP_PROFILE \ + -Pkubernetes \ + -Pkubernetes-integration-tests \ + ${properties[@]} +) From 167b681496eaea4a98c4c973bf479c1639863179 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Tue, 7 Dec 2021 03:51:15 +0900 Subject: [PATCH 3/3] Use own realpath script function. --- .../integration-tests/dev/dev-run-integration-tests.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index b3f87088ef5e..2bee2a1ab992 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -19,6 +19,8 @@ set -exo errexit TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +. $TEST_ROOT_DIR/build/util.sh + DEPLOY_MODE="minikube" IMAGE_REPO="docker.io/kubespark" SPARK_TGZ="N/A" @@ -150,10 +152,7 @@ fi if [ -n "$DOCKER_FILE" ]; then - properties=( - ${properties[@]} - -Dspark.kubernetes.test.dockerFile=$( - python3 -c "import os.path; print(os.path.realpath(\"$DOCKER_FILE\"))") ) + properties=( ${properties[@]} -Dspark.kubernetes.test.dockerFile=$(realpath $DOCKER_FILE) ) fi if [ -n "$NAMESPACE" ];