Rename spark.shuffle.service.name to spark.yarn.shuffle.service.name and move that option to appropriate place

Author: Alexander Bessonov

common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java

@@ -139,7 +139,7 @@ public YarnShuffleService() {
     this("spark_shuffle");
   }
 
-  public YarnShuffleService(String serviceName) {
+  protected YarnShuffleService(String serviceName) {
     super(serviceName);
     logger.info("Initializing YARN shuffle service for Spark");
     instance = this;

core/src/main/scala/org/apache/spark/internal/config/package.scala

@@ -443,9 +443,6 @@ package object config {
   private[spark] val SHUFFLE_SERVICE_PORT =
     ConfigBuilder("spark.shuffle.service.port").intConf.createWithDefault(7337)
 
-  private[spark] val SHUFFLE_SERVICE_NAME =
-    ConfigBuilder("spark.shuffle.service.name").stringConf.createWithDefault("spark_shuffle")
-
   private[spark] val KEYTAB = ConfigBuilder("spark.kerberos.keytab")
     .doc("Location of user's keytab.")
     .stringConf.createOptional

docs/configuration.md

@@ -865,13 +865,6 @@ Apart from these, the following properties are also available, and may be useful
     configuration and setup documentation</a> for more information.
   </td>
 </tr>
-<tr>
-  <td><code>spark.shuffle.service.name</code></td>
-  <td>spark_shuffle</td>
-  <td>
-    Name of the external shuffle service.
-  </td>
-</tr>
 <tr>
   <td><code>spark.shuffle.service.port</code></td>
   <td>7337</td>

docs/running-on-yarn.md

@@ -492,6 +492,13 @@ To use a custom metrics.properties for the application master and executors, upd
   If it is not set then the YARN application ID is used.
   </td>
 </tr>
+<tr>
+  <td><code>spark.yarn.shuffle.service.name</code></td>
+  <td><code>spark_shuffle</code></td>
+  <td>
+    Name of the external shuffle service.
+  </td>
+</tr>
 </table>
 
 #### Available patterns for SHS custom executor log URL

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala

@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
 import org.apache.hadoop.yarn.util.Records
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkException}
+import org.apache.spark.deploy.yarn.config._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.network.util.JavaUtils
@@ -115,7 +116,7 @@ private[yarn] class ExecutorRunnable(
           // Authentication is not enabled, so just provide dummy metadata
           ByteBuffer.allocate(0)
         }
-      val serviceName = sparkConf.get(SHUFFLE_SERVICE_NAME)
+      val serviceName = sparkConf.get(YARN_SHUFFLE_SERVICE_NAME)
       ctx.setServiceData(Collections.singletonMap(serviceName, secretBytes))
     }
 

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala

@@ -320,4 +320,8 @@ package object config {
   private[yarn] val YARN_DRIVER_RESOURCE_TYPES_PREFIX = "spark.yarn.driver.resource."
   private[yarn] val YARN_AM_RESOURCE_TYPES_PREFIX = "spark.yarn.am.resource."
 
+
+  private[yarn] val YARN_SHUFFLE_SERVICE_NAME = ConfigBuilder("spark.yarn.shuffle.service.name")
+    .stringConf
+    .createWithDefault("spark_shuffle")
 }

resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala

@@ -55,7 +55,7 @@ class YarnShuffleIntegrationSuite extends BaseYarnClusterSuite {
     logInfo("Shuffle service port = " + shuffleServicePort)
 
     Map(
-      SHUFFLE_SERVICE_NAME.key -> "spark2_shuffle",
+      YARN_SHUFFLE_SERVICE_NAME.key -> "spark2_shuffle",
       SHUFFLE_SERVICE_ENABLED.key -> "true",
       SHUFFLE_SERVICE_PORT.key -> shuffleServicePort.toString,
       MAX_EXECUTOR_FAILURES.key -> "1"