From 803e93e079dd4fe247265b25f93db11537bfc699 Mon Sep 17 00:00:00 2001 From: "shekhar.bansal" Date: Mon, 27 Apr 2015 16:53:46 +0530 Subject: [PATCH 1/5] [SPARK-6653][yarn] New configuration property to specify port for sparkYarnAM actor system --- .../scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 93ae45133ce24..3d9ae595aa10d 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -274,7 +274,8 @@ private[spark] class ApplicationMaster( } private def runExecutorLauncher(securityMgr: SecurityManager): Unit = { - rpcEnv = RpcEnv.create("sparkYarnAM", Utils.localHostName, 0, sparkConf, securityMgr) + val port = sparkConf.getInt("spark.yarn.am.port", 0) + rpcEnv = RpcEnv.create("sparkYarnAM", Utils.localHostName, port, sparkConf, securityMgr) waitForSparkDriver() addAmIpFilter() registerAM(sparkConf.get("spark.driver.appUIAddress", ""), securityMgr) From 456a59249a49d35d89cc7db4e53b1d0b39fa174d Mon Sep 17 00:00:00 2001 From: "shekhar.bansal" Date: Mon, 27 Apr 2015 17:24:09 +0530 Subject: [PATCH 2/5] [SPARK-6653][yarn] New configuration property to specify port for sparkYarnAM actor system --- docs/running-on-yarn.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 0968fc5ad632b..513bbecdaaae8 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -133,6 +133,13 @@ Most of the configs are the same for Spark on YARN as for other deployment modes Same as spark.yarn.driver.memoryOverhead, but for the Application Master in client mode. + + spark.yarn.am.port + (random) + + Port for the YARN Application Master to listen on. + + spark.yarn.queue default From 9de53309551960fde7395074bff12279e00d0fb5 Mon Sep 17 00:00:00 2001 From: "shekhar.bansal" Date: Wed, 29 Apr 2015 12:29:56 +0530 Subject: [PATCH 3/5] [SPARK-6653][yarn] New config to specify port for sparkYarnAM actor system --- docs/running-on-yarn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 513bbecdaaae8..1b026a9f98138 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -137,7 +137,7 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.am.port (random) - Port for the YARN Application Master to listen on. + Port for the YARN Application Master to listen on. This is used for communicating with the executors. From 51172580913becbf3c3d5462476217f24c68467c Mon Sep 17 00:00:00 2001 From: "shekhar.bansal" Date: Sat, 2 May 2015 14:03:20 +0530 Subject: [PATCH 4/5] [SPARK-6653][yarn] New config to specify port for sparkYarnAM actor system updated config description as suggested by @tgravescs --- docs/running-on-yarn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 1b026a9f98138..6af00b164369d 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -137,7 +137,7 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.am.port (random) - Port for the YARN Application Master to listen on. This is used for communicating with the executors. + Port for the YARN Application Master to listen on. In yarn client mode, this is used to communicate between the spark driver running on a gateway and the application master running on yarn. In yarn cluster mode, this is used for the dynamic executor feature, where it handles the kill from the scheduler backend. From 5574ff72d51f5a5cba3e8c8524207eb4dca15c34 Mon Sep 17 00:00:00 2001 From: "shekhar.bansal" Date: Sun, 3 May 2015 14:20:53 +0530 Subject: [PATCH 5/5] [SPARK-6653][yarn] New config to specify port for sparkYarnAM actor system Incorporating @srowen feedback --- docs/running-on-yarn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 6af00b164369d..989b0b028ce7e 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -137,7 +137,7 @@ Most of the configs are the same for Spark on YARN as for other deployment modes spark.yarn.am.port (random) - Port for the YARN Application Master to listen on. In yarn client mode, this is used to communicate between the spark driver running on a gateway and the application master running on yarn. In yarn cluster mode, this is used for the dynamic executor feature, where it handles the kill from the scheduler backend. + Port for the YARN Application Master to listen on. In YARN client mode, this is used to communicate between the Spark driver running on a gateway and the Application Master running on YARN. In YARN cluster mode, this is used for the dynamic executor feature, where it handles the kill from the scheduler backend.