diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala index 8e3dcdc9d378..b9ad4253e7de 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala @@ -174,6 +174,11 @@ private[rest] class StandaloneSubmitRequestServlet( conf: SparkConf) extends SubmitRequestServlet { + private def replacePlaceHolder(variable: String) = variable match { + case s"{{$name}}" if System.getenv(name) != null => System.getenv(name) + case _ => variable + } + /** * Build a driver description from the fields specified in the submit request. * @@ -214,8 +219,10 @@ private[rest] class StandaloneSubmitRequestServlet( _.replace(s":$masterRestPort", s":$masterPort")).getOrElse(masterUrl) val appArgs = request.appArgs // Filter SPARK_LOCAL_(IP|HOSTNAME) environment variables from being set on the remote system. + // In addition, the placeholders are replaced into the values of environment variables. val environmentVariables = request.environmentVariables.filterNot(x => x._1.matches("SPARK_LOCAL_(IP|HOSTNAME)")) + .map(x => (x._1, replacePlaceHolder(x._2))) // Construct driver description val conf = new SparkConf(false) diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala index 43a5f7a784f5..f8b595fd4925 100644 --- a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala @@ -445,6 +445,18 @@ class StandaloneRestSubmitSuite extends SparkFunSuite { assert(filteredVariables == Map("SPARK_VAR" -> "1")) } + test("SPARK-49033: Support server-side environment variable replacement in REST Submission API") { + val request = new CreateSubmissionRequest + request.appResource = "" + request.mainClass = "" + request.appArgs = Array.empty[String] + request.sparkProperties = Map.empty[String, String] + request.environmentVariables = Map("AWS_ENDPOINT_URL" -> "{{SPARK_SCALA_VERSION}}") + val servlet = new StandaloneSubmitRequestServlet(null, null, null) + val desc = servlet.buildDriverDescription(request, "spark://master:7077", 6066) + assert(desc.command.environment.get("AWS_ENDPOINT_URL") === Some("2.13")) + } + test("SPARK-45197: Make StandaloneRestServer add JavaModuleOptions to drivers") { val request = new CreateSubmissionRequest request.appResource = ""