diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f592ecd522003..d9e63e86bec5e 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -639,13 +639,19 @@ object KubernetesIntegrationTests { if (shouldBuildImage) { val dockerTool = s"$sparkHome/bin/docker-image-tool.sh" val bindingsDir = s"$sparkHome/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings" + val dockerFile = sys.props.get("spark.kubernetes.test.dockerFile") + val javaImageTag = sys.props.getOrElse("spark.kubernetes.test.javaImageTag", "8-jre-slim") + val extraOptions = if (dockerFile.isDefined) { + Seq("-f", s"${dockerFile.get}") + } else { + Seq("-b", s"java_image_tag=$javaImageTag") + } val cmd = Seq(dockerTool, "-m", "-t", imageTag.value, "-p", s"$bindingsDir/python/Dockerfile", - "-R", s"$bindingsDir/R/Dockerfile", - "-b", s"java_image_tag=${sys.env.getOrElse("JAVA_IMAGE_TAG", "8-jre-slim")}", + "-R", s"$bindingsDir/R/Dockerfile") ++ + extraOptions :+ "build" - ) val ec = Process(cmd).! if (ec != 0) { throw new IllegalStateException(s"Process '${cmd.mkString(" ")}' exited with $ec.") diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index d281e38ebf05d..74d4ade785068 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -43,6 +43,7 @@ + N/A @@ -112,6 +113,9 @@ --spark-tgz ${spark.kubernetes.test.sparkTgz} + --docker-file + ${spark.kubernetes.test.dockerFile} + --test-exclude-tags "${test.exclude.tags}" diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh index beda56cf37c94..562d1d820cdd1 100755 --- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -26,6 +26,7 @@ IMAGE_TAG="N/A" JAVA_IMAGE_TAG="8-jre-slim" SPARK_TGZ="N/A" MVN="$TEST_ROOT_DIR/build/mvn" +DOCKER_FILE="N/A" EXCLUDE_TAGS="" # Parse arguments @@ -59,6 +60,10 @@ while (( "$#" )); do SPARK_TGZ="$2" shift ;; + --docker-file) + DOCKER_FILE="$2" + shift + ;; --test-exclude-tags) EXCLUDE_TAGS="$2" shift @@ -97,8 +102,12 @@ then IMAGE_TAG=${VERSION}_$(uuidgen) cd $SPARK_INPUT_DIR - # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim) - JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG" + if [[ $DOCKER_FILE == "N/A" ]]; then + # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim) + JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG" + else + JAVA_IMAGE_TAG_BUILD_ARG="-f $DOCKER_FILE" + fi # Build PySpark image LANGUAGE_BINDING_BUILD_ARGS="-p $DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"