From 20c991c4f0e5b4cd59247ab018dcac52b7c1020e Mon Sep 17 00:00:00 2001 From: summaryzb Date: Fri, 12 Apr 2024 17:15:39 +0800 Subject: [PATCH] [FEATURE] Support dynamic allocation for Gluten Uniffle --- patch/spark/spark-3.2.1_dynamic_allocation_support.patch | 7 +++++-- patch/spark/spark-3.3.1_dynamic_allocation_support.patch | 9 ++++++--- patch/spark/spark-3.4.1_dynamic_allocation_support.patch | 7 +++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/patch/spark/spark-3.2.1_dynamic_allocation_support.patch b/patch/spark/spark-3.2.1_dynamic_allocation_support.patch index 81046dec0f..815f7bdc65 100644 --- a/patch/spark/spark-3.2.1_dynamic_allocation_support.patch +++ b/patch/spark/spark-3.2.1_dynamic_allocation_support.patch @@ -34,14 +34,17 @@ diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/mai index 5f37a1abb19..af4bee1e1bb 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala -@@ -580,6 +580,10 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria +@@ -580,6 +580,13 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria Utils.redact(this, getAll).sorted.map { case (k, v) => k + "=" + v }.mkString("\n") } + /** + * Return true if remote shuffle service is enabled. + */ -+ def isRssEnable(): Boolean = get("spark.shuffle.manager", "sort").contains("RssShuffleManager") ++ def isRssEnable(): Boolean = { ++ val shuffleMgr = get("spark.shuffle.manager", "sort") ++ shuffleMgr.contains("RssShuffleManager") || shuffleMgr.contains("UniffleShuffleManager") ++ } } private[spark] object SparkConf extends Logging { diff --git a/patch/spark/spark-3.3.1_dynamic_allocation_support.patch b/patch/spark/spark-3.3.1_dynamic_allocation_support.patch index 194e873cd3..517555d810 100644 --- a/patch/spark/spark-3.3.1_dynamic_allocation_support.patch +++ b/patch/spark/spark-3.3.1_dynamic_allocation_support.patch @@ -34,14 +34,17 @@ diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/mai index 5f37a1abb19..af4bee1e1bb 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala -@@ -580,6 +580,10 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria +@@ -580,6 +580,13 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria Utils.redact(this, getAll).sorted.map { case (k, v) => k + "=" + v }.mkString("\n") } - + + /** + * Return true if remote shuffle service is enabled. + */ -+ def isRssEnable(): Boolean = get("spark.shuffle.manager", "sort").contains("RssShuffleManager") ++ def isRssEnable(): Boolean = { ++ val shuffleMgr = get("spark.shuffle.manager", "sort") ++ shuffleMgr.contains("RssShuffleManager") || shuffleMgr.contains("UniffleShuffleManager") ++ } } private[spark] object SparkConf extends Logging { diff --git a/patch/spark/spark-3.4.1_dynamic_allocation_support.patch b/patch/spark/spark-3.4.1_dynamic_allocation_support.patch index 2f79bba3b2..0ea65123e1 100644 --- a/patch/spark/spark-3.4.1_dynamic_allocation_support.patch +++ b/patch/spark/spark-3.4.1_dynamic_allocation_support.patch @@ -34,14 +34,17 @@ diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/mai index 08344d8e547..ff3bab6710d 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala -@@ -580,6 +580,10 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria +@@ -580,6 +580,13 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria Utils.redact(this, getAll).sorted.map { case (k, v) => k + "=" + v }.mkString("\n") } + /** + * Return true if remote shuffle service is enabled. + */ -+ def isRssEnable(): Boolean = get("spark.shuffle.manager", "sort").contains("RssShuffleManager") ++ def isRssEnable(): Boolean = { ++ val shuffleMgr = get("spark.shuffle.manager", "sort") ++ shuffleMgr.contains("RssShuffleManager") || shuffleMgr.contains("UniffleShuffleManager") ++ } } private[spark] object SparkConf extends Logging {