diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 145440d43ede..0c887e0e70ed 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1137,6 +1137,7 @@ object CopyDependencies { object TestSettings { import BuildCommons._ private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest", + "org.apache.spark.deploy.k8s.integrationtest.YuniKornTag", "org.apache.spark.internal.io.cloud.IntegrationTestSuite") lazy val settings = Seq ( diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 5f94203c0e2d..f5f93adeddf6 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -37,6 +37,7 @@ SERVICE_ACCOUNT= CONTEXT= INCLUDE_TAGS="k8s" EXCLUDE_TAGS= +DEFAULT_EXCLUDE_TAGS="N/A" JAVA_VERSION="8" BUILD_DEPENDENCIES_MVN_FLAG="-am" HADOOP_PROFILE="hadoop-3" @@ -101,6 +102,10 @@ while (( "$#" )); do EXCLUDE_TAGS="$2" shift ;; + --default-exclude-tags) + DEFAULT_EXCLUDE_TAGS="$2" + shift + ;; --base-image-name) BASE_IMAGE_NAME="$2" shift @@ -180,6 +185,11 @@ then properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS ) fi +if [ "$DEFAULT_EXCLUDE_TAGS" != "N/A" ]; +then + properties=( ${properties[@]} -Dtest.default.exclude.tags=$DEFAULT_EXCLUDE_TAGS ) +fi + BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-spark} JVM_IMAGE_NAME=${JVM_IMAGE_NAME:-${BASE_IMAGE_NAME}} PYTHON_IMAGE_NAME=${PYTHON_IMAGE_NAME:-${BASE_IMAGE_NAME}-py} diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index e468523d254b..7580982c2330 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -46,6 +46,7 @@ Dockerfile.java17 + org.apache.spark.deploy.k8s.integrationtest.YuniKornTag **/*Volcano*.scala @@ -137,7 +138,7 @@ ${spark.kubernetes.test.dockerFile} --test-exclude-tags - "${test.exclude.tags}" + "${test.exclude.tags},${test.default.exclude.tags}" @@ -179,7 +180,7 @@ ${spark.kubernetes.test.pythonImage} ${spark.kubernetes.test.rImage} - ${test.exclude.tags} + ${test.exclude.tags},${test.default.exclude.tags} ${test.include.tags} diff --git a/resource-managers/kubernetes/integration-tests/src/test/java/org/apache/spark/deploy/k8s/integrationtest/YuniKornTag.java b/resource-managers/kubernetes/integration-tests/src/test/java/org/apache/spark/deploy/k8s/integrationtest/YuniKornTag.java new file mode 100644 index 000000000000..cc21cad7aad1 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/java/org/apache/spark/deploy/k8s/integrationtest/YuniKornTag.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.ElementType; + +@org.scalatest.TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface YuniKornTag {} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala new file mode 100644 index 000000000000..5a3c063efa14 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +@YuniKornTag +class YuniKornSuite extends KubernetesSuite { + + override protected def setUpTest(): Unit = { + super.setUpTest() + sparkAppConf + .set("spark.kubernetes.scheduler.name", "yunikorn") + .set("spark.kubernetes.driver.annotation.yunikorn.apache.org/app-id", "{{APP_ID}}") + .set("spark.kubernetes.executor.annotation.yunikorn.apache.org/app-id", "{{APP_ID}}") + } +}