From 02a041a64d16f14185a2a698a96bdbabc4419511 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Thu, 13 Jul 2017 14:13:12 -0700 Subject: [PATCH 1/4] Add spnego auth support for ThriftServer thrift/http protocol Change-Id: Icec0957ae47cb031883a3c9affcea8580cf9ab35 --- .../thriftserver/SparkSQLCLIService.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala index 1b17a9a56e5b..5d56a8421c78 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala @@ -25,6 +25,7 @@ import scala.collection.JavaConverters._ import org.apache.commons.logging.Log import org.apache.hadoop.hive.conf.HiveConf +import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.hadoop.hive.shims.Utils import org.apache.hadoop.security.UserGroupInformation import org.apache.hive.service.{AbstractService, Service, ServiceException} @@ -47,6 +48,7 @@ private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLC setSuperField(this, "sessionManager", sparkSqlSessionManager) addService(sparkSqlSessionManager) var sparkServiceUGI: UserGroupInformation = null + var httpUGI: UserGroupInformation = null if (UserGroupInformation.isSecurityEnabled) { try { @@ -57,6 +59,23 @@ private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLC case e @ (_: IOException | _: LoginException) => throw new ServiceException("Unable to login to kerberos with given principal/keytab", e) } + + // Try creating spnego UGI if it is configured. + val principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL) + val keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB) + if (principal.isEmpty || keyTabFile.isEmpty) { + getAncestorField[Log](this, 3, "LOG").info( + s"SPNego httpUGI not created, spNegoPrincipal: $principal , ketabFile: $keyTabFile") + } else { + try { + httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf) + setSuperField(this, "httpUGI", httpUGI) + getAncestorField[Log](this, 3, "LOG").info("SPNego httpUGI successfully created.") + } catch { + case e: IOException => + getAncestorField[Log](this, 3, "LOG").warn(s"SPNego httpUGI creation failed: $e") + } + } } initCompositeService(hiveConf) From 787e72c419bb1ea29af6c7eee6b9c71d02d24a57 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Fri, 14 Jul 2017 15:14:58 -0700 Subject: [PATCH 2/4] Avoid using reflection Log object Change-Id: I801f048afc6f683f7519b0cbf1c99e2268500c45 --- .../spark/sql/hive/thriftserver/SparkSQLCLIService.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala index 5d56a8421c78..f9a7c5550a13 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala @@ -63,17 +63,14 @@ private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLC // Try creating spnego UGI if it is configured. val principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL) val keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB) - if (principal.isEmpty || keyTabFile.isEmpty) { - getAncestorField[Log](this, 3, "LOG").info( - s"SPNego httpUGI not created, spNegoPrincipal: $principal , ketabFile: $keyTabFile") - } else { + if (principal.nonEmpty && keyTabFile.nonEmpty) { try { httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf) setSuperField(this, "httpUGI", httpUGI) - getAncestorField[Log](this, 3, "LOG").info("SPNego httpUGI successfully created.") } catch { case e: IOException => - getAncestorField[Log](this, 3, "LOG").warn(s"SPNego httpUGI creation failed: $e") + throw new ServiceException("Unable to login to spnego with given principal/keytab " + + s"$principal/$keyTabFile", e) } } } From 47653983939650b5987fc6716cd49f6f577c063f Mon Sep 17 00:00:00 2001 From: jerryshao Date: Mon, 17 Jul 2017 13:52:20 -0700 Subject: [PATCH 3/4] Address the comments Change-Id: I486c361a96687e3611754895d895b5326ee22f52 --- .../spark/sql/hive/thriftserver/SparkSQLCLIService.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala index f9a7c5550a13..4aab3c561862 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala @@ -69,8 +69,7 @@ private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLC setSuperField(this, "httpUGI", httpUGI) } catch { case e: IOException => - throw new ServiceException("Unable to login to spnego with given principal/keytab " + - s"$principal/$keyTabFile", e) + throw new ServiceException("Unable to login to spnego with given principal/keytab", e) } } } From 12565cdc8a11761d3b6e383807a873496a8e7f0d Mon Sep 17 00:00:00 2001 From: jerryshao Date: Thu, 3 Aug 2017 14:36:18 +0800 Subject: [PATCH 4/4] Address the comments Change-Id: Icce88ff9b266f470fd5407496756c933c0f1f959 --- .../spark/sql/hive/thriftserver/SparkSQLCLIService.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala index 4aab3c561862..ad1f5eb9ca3a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala @@ -61,15 +61,16 @@ private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, sqlContext: SQLC } // Try creating spnego UGI if it is configured. - val principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL) - val keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB) + val principal = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_PRINCIPAL).trim + val keyTabFile = hiveConf.getVar(ConfVars.HIVE_SERVER2_SPNEGO_KEYTAB).trim if (principal.nonEmpty && keyTabFile.nonEmpty) { try { httpUGI = HiveAuthFactory.loginFromSpnegoKeytabAndReturnUGI(hiveConf) setSuperField(this, "httpUGI", httpUGI) } catch { case e: IOException => - throw new ServiceException("Unable to login to spnego with given principal/keytab", e) + throw new ServiceException("Unable to login to spnego with given principal " + + s"$principal and keytab $keyTabFile: $e", e) } } }