From 295cf3b288ffa7ecdbab93bfa4c72a45b77260a0 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 14 Dec 2022 23:19:43 -0800 Subject: [PATCH] [SPARK-41525][K8S] Use unique list of known executor IDs and PVC names in onNewSnapshots --- .../spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala index f25a0171205b..52821a519dd0 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala @@ -152,7 +152,7 @@ class ExecutorPodsAllocator( applicationId: String, schedulerBackend: KubernetesClusterSchedulerBackend, snapshots: Seq[ExecutorPodsSnapshot]): Unit = { - val k8sKnownExecIds = snapshots.flatMap(_.executorPods.keys) + val k8sKnownExecIds = snapshots.flatMap(_.executorPods.keys).distinct newlyCreatedExecutors --= k8sKnownExecIds schedulerKnownNewlyCreatedExecs --= k8sKnownExecIds @@ -162,7 +162,7 @@ class ExecutorPodsAllocator( val k8sKnownPVCNames = snapshots.flatMap(_.executorPods.values.map(_.pod)).flatMap { pod => pod.getSpec.getVolumes.asScala .flatMap { v => Option(v.getPersistentVolumeClaim).map(_.getClaimName) } - } + }.distinct // transfer the scheduler backend known executor requests from the newlyCreatedExecutors // to the schedulerKnownNewlyCreatedExecs