Removed references to spark.shuffle.service.enabled = true (#145)

best-practice-and-impact · Oct 24, 2024 · f75b0cc · f75b0cc
1 parent eeb25b6
commit f75b0cc
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 12 deletions.
diff --git a/ons-spark/spark-overview/example-spark-sessions.md b/ons-spark/spark-overview/example-spark-sessions.md
@@ -112,7 +112,6 @@ spark = (
     .config("spark.dynamicAllocation.enabled", "true")
     .config("spark.dynamicAllocation.maxExecutors", 3)
     .config("spark.sql.shuffle.partitions", 12)
-    .config("spark.shuffle.service.enabled", "true")
     .config("spark.ui.showConsoleProgress", "false")
     .enableHiveSupport()
     .getOrCreate()
@@ -127,7 +126,6 @@ small_config$spark.executor.cores <- 1
 small_config$spark.dynamicAllocation.enabled <- "true"
 small_config$spark.dynamicAllocation.maxExecutors <- 3
 small_config$spark.sql.shuffle.partitions <- 12
-small_config$spark.shuffle.service.enabled <- "true"
 
 sc <- sparklyr::spark_connect(
   master = "yarn-client",
@@ -164,7 +162,6 @@ spark = (
         .config("spark.dynamicAllocation.enabled", "true")
         .config("spark.dynamicAllocation.maxExecutors", 3)
         .config("spark.sql.shuffle.partitions", 18)
-        .config("spark.shuffle.service.enabled", "true")
         .config("spark.ui.showConsoleProgress", "false")
         .enableHiveSupport()
         .getOrCreate()
@@ -179,7 +176,6 @@ medium_config$spark.executor.cores <- 3
 medium_config$spark.dynamicAllocation.enabled <- "true"
 medium_config$spark.dynamicAllocation.maxExecutors <- 3
 medium_config$spark.sql.shuffle.partitions <- 18
-medium_config$spark.shuffle.service.enabled <- "true"
 
 sc <- sparklyr::spark_connect(
   master = "yarn-client",
@@ -218,7 +214,6 @@ spark = (
     .config("spark.dynamicAllocation.enabled", "true")
     .config("spark.dynamicAllocation.maxExecutors", 5)
     .config("spark.sql.shuffle.partitions", 200)
-    .config("spark.shuffle.service.enabled", "true")
     .config("spark.ui.showConsoleProgress", "false")
     .enableHiveSupport()
     .getOrCreate()
@@ -234,7 +229,6 @@ large_config$spark.executor.cores <- 5
 large_config$spark.dynamicAllocation.enabled <- "true"
 large_config$spark.dynamicAllocation.maxExecutors <- 5
 large_config$spark.sql.shuffle.partitions <- 200
-large_config$spark.shuffle.service.enabled <- "true"
 
 sc <- sparklyr::spark_connect(
   master = "yarn-client",
@@ -275,7 +269,6 @@ spark = (
     .config("spark.dynamicAllocation.enabled", "true")
     .config("spark.dynamicAllocation.maxExecutors", 12)
     .config("spark.sql.shuffle.partitions", 240)
-    .config("spark.shuffle.service.enabled", "true")
     .config("spark.ui.showConsoleProgress", "false")
     .enableHiveSupport()
     .getOrCreate()
@@ -291,7 +284,6 @@ xl_config$spark.executor.cores <- 5
 xl_config$spark.dynamicAllocation.enabled <- "true"
 xl_config$spark.dynamicAllocation.maxExecutors <- 12
 xl_config$spark.sql.shuffle.partitions <- 240
-xl_config$spark.shuffle.service.enabled <- "true"
 
 sc <- sparklyr::spark_connect(
   master = "yarn-client",

diff --git a/ons-spark/spark-overview/spark-defaults.md b/ons-spark/spark-overview/spark-defaults.md
@@ -48,7 +48,6 @@ spark.executor.cores                 1
 spark.dynamicAllocation.enabled      true
 spark.dynamicAllocation.maxExecutors 3
 spark.sql.shuffle.partitions         12
-spark.shuffle.service.enabled        true
 spark.ui.showConsoleProgress         false
 ```
 

diff --git a/ons-spark/spark-overview/spark-session-guidance.md b/ons-spark/spark-overview/spark-session-guidance.md
@@ -137,8 +137,8 @@ Also note when specifying this config, Spark will reserve the maxExecutors for y
 stop your session- even if you don't need them. See the [reserving](#reserving) section of the calculations below for more details.
 
 **Shuffle Service**  
-`.config("spark.shuffle.service.enabled", "true")`  
-This property helps with processes like switching executors off safely using dynamic allocation by making use of shuffle files.
+`.config("spark.shuffle.service.enabled", "false")`  
+This property helps with processes like switching executors off safely using dynamic allocation by making use of shuffle files. The default value of this property is false.
 
 **Hive Support**  
 `.enableHiveSupport()`  
@@ -164,7 +164,6 @@ spark = (
     .config("spark.executor.cores", 5)
     .config("spark.dynamicAllocation.enabled", "true")
     .config("spark.dynamicAllocation.maxExecutors", 5)
-    .config("spark.shuffle.service.enabled", "true")
     .config("spark.ui.showConsoleProgress", "false")
     .enableHiveSupport()
     .getOrCreate()