diff --git a/ons-spark/spark-overview/example-spark-sessions.md b/ons-spark/spark-overview/example-spark-sessions.md index 58cc9dda..0bf09a09 100644 --- a/ons-spark/spark-overview/example-spark-sessions.md +++ b/ons-spark/spark-overview/example-spark-sessions.md @@ -112,7 +112,6 @@ spark = ( .config("spark.dynamicAllocation.enabled", "true") .config("spark.dynamicAllocation.maxExecutors", 3) .config("spark.sql.shuffle.partitions", 12) - .config("spark.shuffle.service.enabled", "true") .config("spark.ui.showConsoleProgress", "false") .enableHiveSupport() .getOrCreate() @@ -127,7 +126,6 @@ small_config$spark.executor.cores <- 1 small_config$spark.dynamicAllocation.enabled <- "true" small_config$spark.dynamicAllocation.maxExecutors <- 3 small_config$spark.sql.shuffle.partitions <- 12 -small_config$spark.shuffle.service.enabled <- "true" sc <- sparklyr::spark_connect( master = "yarn-client", @@ -164,7 +162,6 @@ spark = ( .config("spark.dynamicAllocation.enabled", "true") .config("spark.dynamicAllocation.maxExecutors", 3) .config("spark.sql.shuffle.partitions", 18) - .config("spark.shuffle.service.enabled", "true") .config("spark.ui.showConsoleProgress", "false") .enableHiveSupport() .getOrCreate() @@ -179,7 +176,6 @@ medium_config$spark.executor.cores <- 3 medium_config$spark.dynamicAllocation.enabled <- "true" medium_config$spark.dynamicAllocation.maxExecutors <- 3 medium_config$spark.sql.shuffle.partitions <- 18 -medium_config$spark.shuffle.service.enabled <- "true" sc <- sparklyr::spark_connect( master = "yarn-client", @@ -218,7 +214,6 @@ spark = ( .config("spark.dynamicAllocation.enabled", "true") .config("spark.dynamicAllocation.maxExecutors", 5) .config("spark.sql.shuffle.partitions", 200) - .config("spark.shuffle.service.enabled", "true") .config("spark.ui.showConsoleProgress", "false") .enableHiveSupport() .getOrCreate() @@ -234,7 +229,6 @@ large_config$spark.executor.cores <- 5 large_config$spark.dynamicAllocation.enabled <- "true" large_config$spark.dynamicAllocation.maxExecutors <- 5 large_config$spark.sql.shuffle.partitions <- 200 -large_config$spark.shuffle.service.enabled <- "true" sc <- sparklyr::spark_connect( master = "yarn-client", @@ -275,7 +269,6 @@ spark = ( .config("spark.dynamicAllocation.enabled", "true") .config("spark.dynamicAllocation.maxExecutors", 12) .config("spark.sql.shuffle.partitions", 240) - .config("spark.shuffle.service.enabled", "true") .config("spark.ui.showConsoleProgress", "false") .enableHiveSupport() .getOrCreate() @@ -291,7 +284,6 @@ xl_config$spark.executor.cores <- 5 xl_config$spark.dynamicAllocation.enabled <- "true" xl_config$spark.dynamicAllocation.maxExecutors <- 12 xl_config$spark.sql.shuffle.partitions <- 240 -xl_config$spark.shuffle.service.enabled <- "true" sc <- sparklyr::spark_connect( master = "yarn-client", diff --git a/ons-spark/spark-overview/spark-defaults.md b/ons-spark/spark-overview/spark-defaults.md index ffd5001c..c4d53ba7 100644 --- a/ons-spark/spark-overview/spark-defaults.md +++ b/ons-spark/spark-overview/spark-defaults.md @@ -48,7 +48,6 @@ spark.executor.cores 1 spark.dynamicAllocation.enabled true spark.dynamicAllocation.maxExecutors 3 spark.sql.shuffle.partitions 12 -spark.shuffle.service.enabled true spark.ui.showConsoleProgress false ``` diff --git a/ons-spark/spark-overview/spark-session-guidance.md b/ons-spark/spark-overview/spark-session-guidance.md index 0954bab4..fb14a5c9 100644 --- a/ons-spark/spark-overview/spark-session-guidance.md +++ b/ons-spark/spark-overview/spark-session-guidance.md @@ -137,8 +137,8 @@ Also note when specifying this config, Spark will reserve the maxExecutors for y stop your session- even if you don't need them. See the [reserving](#reserving) section of the calculations below for more details. **Shuffle Service** -`.config("spark.shuffle.service.enabled", "true")` -This property helps with processes like switching executors off safely using dynamic allocation by making use of shuffle files. +`.config("spark.shuffle.service.enabled", "false")` +This property helps with processes like switching executors off safely using dynamic allocation by making use of shuffle files. The default value of this property is false. **Hive Support** `.enableHiveSupport()` @@ -164,7 +164,6 @@ spark = ( .config("spark.executor.cores", 5) .config("spark.dynamicAllocation.enabled", "true") .config("spark.dynamicAllocation.maxExecutors", 5) - .config("spark.shuffle.service.enabled", "true") .config("spark.ui.showConsoleProgress", "false") .enableHiveSupport() .getOrCreate()