respect the --hiveconf in the SparkSQLCliDriver commandline

Author: chenghao-intel

sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala

@@ -156,6 +156,9 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     // Execute -i init files (always in silent mode)
     cli.processInitFiles(sessionState)
 
+    // Respect the configurations from the command line and .hiverc for backward-compatible
+    SparkSQLEnv.applyOverridedConf(sessionState)
+
     if (sessionState.execString != null) {
       System.exit(cli.processLine(sessionState.execString))
     }

sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala

@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.io.PrintStream
 
-import scala.collection.JavaConverters._
+import org.apache.hadoop.hive.ql.session.SessionState
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.internal.Logging
@@ -66,6 +66,14 @@ private[hive] object SparkSQLEnv extends Logging {
     }
   }
 
+  def applyOverridedConf(ss: SessionState): Unit = {
+    val it = ss.getOverriddenConfigurations.entrySet().iterator()
+    while (it.hasNext) {
+      val kv = it.next()
+      SparkSQLEnv.sqlContext.setConf(kv.getKey, kv.getValue)
+    }
+  }
+
   /** Cleans up and shuts down the Spark SQL environments. */
   def stop() {
     logDebug("Shutting down Spark SQL Environment")

sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala

@@ -91,6 +91,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
          |  --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$jdbcUrl
          |  --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
          |  --hiveconf ${ConfVars.SCRATCHDIR}=$scratchDirPath
+         |  --hiveconf conf1=conftest
+         |  --hiveconf conf2=1
        """.stripMargin.split("\\s+").toSeq ++ extraArgs
     }
 
@@ -272,4 +274,13 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       s"LIST FILE $dataFilePath;" -> "small_kv.txt"
     )
   }
+
+  test("apply hiveconf from cli command") {
+    runCliWithin(2.minute)(
+      s"SET conf1;" -> "conftest",
+      s"SET conf2;" -> "1",
+      // bypassed by SparkSQLCLIDriver
+      s"SET ${ConfVars.METASTORECONNECTURLKEY};" -> "undefined"
+    )
+  }
 }