From f6ed84ee2829e8e1c4e9a33afe43e6ac7b88fe63 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Fri, 25 Feb 2022 16:44:02 -0800 Subject: [PATCH 1/8] [SPARK-37809] Add yunikorn feature step --- .github/workflows/build_and_test.yml | 4 +- dev/scalastyle | 2 +- project/SparkBuild.scala | 11 ++++ resource-managers/kubernetes/core/pom.xml | 8 +++ .../k8s/features/YuniKornFeatureStep.scala | 46 ++++++++++++++ .../features/YuniKornFeatureStepSuite.scala | 43 +++++++++++++ .../kubernetes/integration-tests/pom.xml | 7 +++ .../k8s/integrationtest/YuniKornSuite.scala | 33 ++++++++++ .../integrationtest/YuniKornTestsSuite.scala | 60 +++++++++++++++++++ 9 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala create mode 100644 resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3a9157010e9b..53b382e6969f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -661,7 +661,7 @@ jobs: export MAVEN_CLI_OPTS="--no-transfer-progress" export JAVA_VERSION=${{ matrix.java }} # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414. - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Pvolcano -Pyunikorn -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=${JAVA_VERSION/-ea} install rm -rf ~/.m2/repository/org/apache/spark scala-213: @@ -707,7 +707,7 @@ jobs: - name: Build with SBT run: | ./dev/change-scala-version.sh 2.13 - ./build/sbt -Pyarn -Pmesos -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pdocker-integration-tests -Pkubernetes-integration-tests -Pspark-ganglia-lgpl -Pscala-2.13 compile Test/compile + ./build/sbt -Pyarn -Pmesos -Pkubernetes -Pvolcano -Pyunikorn -Phive -Phive-thriftserver -Phadoop-cloud -Pkinesis-asl -Pdocker-integration-tests -Pkubernetes-integration-tests -Pspark-ganglia-lgpl -Pscala-2.13 compile Test/compile # Any TPC-DS related updates on this job need to be applied to tpcds-1g-gen job of benchmark.yml as well tpcds-1g: diff --git a/dev/scalastyle b/dev/scalastyle index 5f958b8fb0a7..374173146ef1 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -17,7 +17,7 @@ # limitations under the License. # -SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive -Pvolcano"} +SPARK_PROFILES=${1:-"-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive -Pvolcano -Pyunikorn"} # NOTE: echo "q" is needed because SBT prompts the user for input on encountering a build file # with failure (either resolution or compilation); the "q" makes SBT quit. diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 21ab6f9f636a..63f4154a4fc9 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -426,6 +426,11 @@ object SparkBuild extends PomBuild { enable(Volcano.settings)(kubernetesIntegrationTests) } + if (!profiles.contains("yunikorn")) { + enable(YuniKorn.settings)(kubernetes) + enable(YuniKorn.settings)(kubernetesIntegrationTests) + } + enable(KubernetesIntegrationTests.settings)(kubernetesIntegrationTests) enable(YARN.settings)(yarn) @@ -973,6 +978,12 @@ object Volcano { ) } +object YuniKorn { + // Exclude all yunikorn file for Compile and Test + lazy val settings = Seq( + unmanagedSources / excludeFilter := HiddenFileFilter || "*YuniKorn*.scala") +} + object Unidoc { import BuildCommons._ diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 1c729cc441ee..4dc0cf9cdba3 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -30,6 +30,7 @@ kubernetes **/*Volcano*.scala + **/*YuniKorn*.scala @@ -51,6 +52,12 @@ + + yunikorn + + + + @@ -134,6 +141,7 @@ ${volcano.exclude} + ${yunikorn.exclude} diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala new file mode 100644 index 000000000000..877a317ce97a --- /dev/null +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.features + +import io.fabric8.kubernetes.api.model._ + +import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverConf, KubernetesExecutorConf, SparkPod} + +private[spark] class YuniKornFeatureStep extends KubernetesDriverCustomFeatureConfigStep + with KubernetesExecutorCustomFeatureConfigStep { + + private var kubernetesConf: KubernetesConf = _ + + private val YUNIKORN_APP_ID_ANNOTATION = "yunikorn.apache.org/app-id" + + override def init(config: KubernetesDriverConf): Unit = { + kubernetesConf = config + } + + override def init(config: KubernetesExecutorConf): Unit = { + kubernetesConf = config + } + + override def configurePod(pod: SparkPod): SparkPod = { + val k8sPodBuilder = new PodBuilder(pod.pod) + .editMetadata() + .addToAnnotations(YUNIKORN_APP_ID_ANNOTATION, kubernetesConf.appId) + .endMetadata() + val k8sPod = k8sPodBuilder.build() + SparkPod(k8sPod, pod.container) + } +} diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala new file mode 100644 index 000000000000..8ae300bf4d35 --- /dev/null +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.features + +import org.apache.spark.{SparkConf, SparkFunSuite} +import org.apache.spark.deploy.k8s._ + +class YuniKornFeatureStepSuite extends SparkFunSuite { + + test("SPARK-37809: Driver Pod with YuniKorn labels") { + val sparkConf = new SparkConf() + val kubernetesConf = KubernetesTestConf.createDriverConf(sparkConf) + val step = new YuniKornFeatureStep() + step.init(kubernetesConf) + val configuredPod = step.configurePod(SparkPod.initialPod()) + val annotations = configuredPod.pod.getMetadata.getAnnotations + assert(annotations.get("yunikorn.apache.org/app-id") === ${kubernetesConf.appId}) + } + + test("SPARK-37809: Executor Pod with YuniKorn labels") { + val sparkConf = new SparkConf() + val kubernetesConf = KubernetesTestConf.createExecutorConf(sparkConf) + val step = new YuniKornFeatureStep() + step.init(kubernetesConf) + val configuredPod = step.configurePod(SparkPod.initialPod()) + val annotations = configuredPod.pod.getMetadata.getAnnotations + assert(annotations.get("yunikorn.apache.org/app-id") === ${kubernetesConf.appId}) + } +} \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index e468523d254b..74332024acad 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -92,6 +92,7 @@ ${volcano.exclude} + ${yunikorn.exclude} @@ -242,5 +243,11 @@ + + yunikorn + + + + diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala new file mode 100644 index 000000000000..6d61e8de6153 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import org.scalatest.Tag + +class YuniKornSuite extends KubernetesSuite with YuniKornTestsSuite { + + override protected def setUpTest(): Unit = { + super.setUpTest() + sparkAppConf + .set("spark.kubernetes.driver.scheduler.name", "yunikorn") + .set("spark.kubernetes.executor.scheduler.name", "yunikorn") + } +} + +private[spark] object YuniKornSuite { + val yunikornTag = Tag("yunikorn") +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala new file mode 100644 index 000000000000..6024bebd57b8 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import io.fabric8.kubernetes.api.model.Pod + +import org.apache.spark.SparkFunSuite +import org.apache.spark.deploy.k8s.features.YuniKornFeatureStep +import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.k8sTestTag +import org.apache.spark.deploy.k8s.integrationtest.YuniKornSuite.yunikornTag + +private[spark] trait YuniKornTestsSuite { k8sSuite: KubernetesSuite => + import YuniKornTestsSuite._ + + protected def checkScheduler(pod: Pod): Unit = { + assert(pod.getSpec.getSchedulerName === "yunikorn") + } + + protected def checkAnnotations(pod: Pod): Unit = { + val appId = pod.getMetadata.getLabels.get("spark-app-selector") + val annotations = pod.getMetadata.getAnnotations + assert(annotations.get("yunikorn.apache.org/app-id") === appId) + } + + test("Run SparkPi with yunikorn scheduler", k8sTestTag, yunikornTag) { + sparkAppConf + .set("spark.kubernetes.driver.pod.featureSteps", YUNIKORN_FEATURE_STEP) + .set("spark.kubernetes.executor.pod.featureSteps", YUNIKORN_FEATURE_STEP) + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkScheduler(driverPod) + checkAnnotations(driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkScheduler(executorPod) + checkAnnotations(executorPod) + } + ) + } +} + +private[spark] object YuniKornTestsSuite extends SparkFunSuite { + val YUNIKORN_FEATURE_STEP = classOf[YuniKornFeatureStep].getName +} From e872fb733d2855e9e3a0686fa9ea8ae62f08cb82 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Sun, 27 Feb 2022 00:09:36 -0800 Subject: [PATCH 2/8] address review comments --- .../deploy/k8s/features/YuniKornFeatureStepSuite.scala | 8 ++++---- resource-managers/kubernetes/integration-tests/pom.xml | 1 + .../spark/deploy/k8s/integrationtest/YuniKornSuite.scala | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala index 8ae300bf4d35..5d842af96570 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala @@ -21,23 +21,23 @@ import org.apache.spark.deploy.k8s._ class YuniKornFeatureStepSuite extends SparkFunSuite { - test("SPARK-37809: Driver Pod with YuniKorn labels") { + test("SPARK-37809: Driver Pod with YuniKorn annotations") { val sparkConf = new SparkConf() val kubernetesConf = KubernetesTestConf.createDriverConf(sparkConf) val step = new YuniKornFeatureStep() step.init(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) val annotations = configuredPod.pod.getMetadata.getAnnotations - assert(annotations.get("yunikorn.apache.org/app-id") === ${kubernetesConf.appId}) + assert(annotations.get("yunikorn.apache.org/app-id") === kubernetesConf.appId) } - test("SPARK-37809: Executor Pod with YuniKorn labels") { + test("SPARK-37809: Executor Pod with YuniKorn annotations") { val sparkConf = new SparkConf() val kubernetesConf = KubernetesTestConf.createExecutorConf(sparkConf) val step = new YuniKornFeatureStep() step.init(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) val annotations = configuredPod.pod.getMetadata.getAnnotations - assert(annotations.get("yunikorn.apache.org/app-id") === ${kubernetesConf.appId}) + assert(annotations.get("yunikorn.apache.org/app-id") === kubernetesConf.appId) } } \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 74332024acad..f52ab9c18a53 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -48,6 +48,7 @@ **/*Volcano*.scala + **/*YuniKorn*.scala jar Spark Project Kubernetes Integration Tests diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala index 6d61e8de6153..a9b8f6327d8b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala @@ -23,8 +23,7 @@ class YuniKornSuite extends KubernetesSuite with YuniKornTestsSuite { override protected def setUpTest(): Unit = { super.setUpTest() sparkAppConf - .set("spark.kubernetes.driver.scheduler.name", "yunikorn") - .set("spark.kubernetes.executor.scheduler.name", "yunikorn") + .set("spark.kubernetes.scheduler.name", "yunikorn") } } From d7205a1387f4d6f5b9167e61f21e7a604829b226 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Sun, 27 Feb 2022 00:18:21 -0800 Subject: [PATCH 3/8] add new line --- .../spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala index 5d842af96570..c3749a809649 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala @@ -40,4 +40,4 @@ class YuniKornFeatureStepSuite extends SparkFunSuite { val annotations = configuredPod.pod.getMetadata.getAnnotations assert(annotations.get("yunikorn.apache.org/app-id") === kubernetesConf.appId) } -} \ No newline at end of file +} From 568a97a45b9f1fd8ee1f9d40968fcbb909cb5c29 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Sun, 27 Feb 2022 19:19:41 -0800 Subject: [PATCH 4/8] fix sbt build --- project/SparkBuild.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 63f4154a4fc9..28e1408c4476 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -981,7 +981,7 @@ object Volcano { object YuniKorn { // Exclude all yunikorn file for Compile and Test lazy val settings = Seq( - unmanagedSources / excludeFilter := HiddenFileFilter || "*YuniKorn*.scala") + unmanagedSources / excludeFilter += HiddenFileFilter || "*YuniKorn*.scala") } object Unidoc { From 6fc3e79f37f2312e7673bf79528378eab774dff9 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Sun, 27 Feb 2022 22:45:07 -0800 Subject: [PATCH 5/8] fix sbt --- project/SparkBuild.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 28e1408c4476..dd2812ad44c0 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -981,7 +981,8 @@ object Volcano { object YuniKorn { // Exclude all yunikorn file for Compile and Test lazy val settings = Seq( - unmanagedSources / excludeFilter += HiddenFileFilter || "*YuniKorn*.scala") + unmanagedSources / excludeFilter ~= { _ || "*YuniKorn*.scala" } + ) } object Unidoc { From ed3e21e1b9b820ee33651e632bdbf2eb630c7dee Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Tue, 1 Mar 2022 00:32:36 -0800 Subject: [PATCH 6/8] address review comments --- .../spark/deploy/k8s/features/YuniKornFeatureStep.scala | 9 ++++++--- .../deploy/k8s/features/YuniKornFeatureStepSuite.scala | 4 ++-- .../spark/deploy/k8s/integrationtest/YuniKornSuite.scala | 4 +++- .../deploy/k8s/integrationtest/YuniKornTestsSuite.scala | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala index 877a317ce97a..61efecf0c6fc 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStep.scala @@ -25,8 +25,6 @@ private[spark] class YuniKornFeatureStep extends KubernetesDriverCustomFeatureCo private var kubernetesConf: KubernetesConf = _ - private val YUNIKORN_APP_ID_ANNOTATION = "yunikorn.apache.org/app-id" - override def init(config: KubernetesDriverConf): Unit = { kubernetesConf = config } @@ -38,9 +36,14 @@ private[spark] class YuniKornFeatureStep extends KubernetesDriverCustomFeatureCo override def configurePod(pod: SparkPod): SparkPod = { val k8sPodBuilder = new PodBuilder(pod.pod) .editMetadata() - .addToAnnotations(YUNIKORN_APP_ID_ANNOTATION, kubernetesConf.appId) + .addToAnnotations(YuniKornFeatureStep.AppIdAnnotationKey, kubernetesConf.appId) .endMetadata() val k8sPod = k8sPodBuilder.build() SparkPod(k8sPod, pod.container) } } + +object YuniKornFeatureStep { + val AppIdAnnotationKey = "yunikorn.apache.org/app-id" + val SchedulerName = "yunikorn" +} diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala index c3749a809649..7e8de76f21a8 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/YuniKornFeatureStepSuite.scala @@ -28,7 +28,7 @@ class YuniKornFeatureStepSuite extends SparkFunSuite { step.init(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) val annotations = configuredPod.pod.getMetadata.getAnnotations - assert(annotations.get("yunikorn.apache.org/app-id") === kubernetesConf.appId) + assert(annotations.get(YuniKornFeatureStep.AppIdAnnotationKey) === kubernetesConf.appId) } test("SPARK-37809: Executor Pod with YuniKorn annotations") { @@ -38,6 +38,6 @@ class YuniKornFeatureStepSuite extends SparkFunSuite { step.init(kubernetesConf) val configuredPod = step.configurePod(SparkPod.initialPod()) val annotations = configuredPod.pod.getMetadata.getAnnotations - assert(annotations.get("yunikorn.apache.org/app-id") === kubernetesConf.appId) + assert(annotations.get(YuniKornFeatureStep.AppIdAnnotationKey) === kubernetesConf.appId) } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala index a9b8f6327d8b..795c1e2429df 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala @@ -18,12 +18,14 @@ package org.apache.spark.deploy.k8s.integrationtest import org.scalatest.Tag +import org.apache.spark.deploy.k8s.features.YuniKornFeatureStep + class YuniKornSuite extends KubernetesSuite with YuniKornTestsSuite { override protected def setUpTest(): Unit = { super.setUpTest() sparkAppConf - .set("spark.kubernetes.scheduler.name", "yunikorn") + .set("spark.kubernetes.scheduler.name", YuniKornFeatureStep.SchedulerName) } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala index 6024bebd57b8..bff2d583abb5 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornTestsSuite.scala @@ -27,13 +27,13 @@ private[spark] trait YuniKornTestsSuite { k8sSuite: KubernetesSuite => import YuniKornTestsSuite._ protected def checkScheduler(pod: Pod): Unit = { - assert(pod.getSpec.getSchedulerName === "yunikorn") + assert(pod.getSpec.getSchedulerName === YuniKornFeatureStep.SchedulerName) } protected def checkAnnotations(pod: Pod): Unit = { val appId = pod.getMetadata.getLabels.get("spark-app-selector") val annotations = pod.getMetadata.getAnnotations - assert(annotations.get("yunikorn.apache.org/app-id") === appId) + assert(annotations.get(YuniKornFeatureStep.AppIdAnnotationKey) === appId) } test("Run SparkPi with yunikorn scheduler", k8sTestTag, yunikornTag) { From 908917942bed35d221df33ee5708c4e35e0bf967 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Fri, 12 Aug 2022 17:38:15 -0700 Subject: [PATCH 7/8] Empty-Commit From 5b686dbde779369df7694008f04869fd683b5ac7 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Fri, 12 Aug 2022 21:49:09 -0700 Subject: [PATCH 8/8] Empty-Commit