From ac8b195fc89ff6da80cd291b8720a8e96622d63a Mon Sep 17 00:00:00 2001 From: fwang12 Date: Wed, 20 Oct 2021 15:03:20 +0800 Subject: [PATCH 01/27] [KYUUBI #1262] Support multiple kinds of SASL authentication type --- docs/deployment/settings.md | 4 +- .../org/apache/kyuubi/config/KyuubiConf.scala | 42 ++++++++++----- .../KyuubiAuthenticationFactory.scala | 52 +++++++++++-------- .../{AuthTypes.scala => PlainAuthTypes.scala} | 6 +-- .../authentication/PlainSASLHelper.scala | 7 ++- .../service/ThriftFrontendServiceSuite.scala | 1 + .../KyuubiAuthenticationFactorySuite.scala | 13 +++-- .../kyuubi/session/SessionManagerSuite.scala | 1 + .../kyuubi/ha/client/ServiceDiscovery.scala | 12 +++-- 9 files changed, 87 insertions(+), 51 deletions(-) rename kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/{AuthTypes.scala => PlainAuthTypes.scala} (88%) diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 69e451c0230..7bb1aa6b8f1 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -122,12 +122,14 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi\.authentication|
NONE
|
Client authentication types.
|
string
|
1.0.0
kyuubi\.authentication
\.custom\.class|
<undefined>
|
User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider
|
string
|
1.3.0
kyuubi\.authentication
\.ldap\.base\.dn|
<undefined>
|
LDAP base DN.
|
string
|
1.0.0
kyuubi\.authentication
\.ldap\.domain|
<undefined>
|
LDAP domain.
|
string
|
1.0.0
kyuubi\.authentication
\.ldap\.guidKey|
uid
|
LDAP attribute name whose values are unique in this LDAP server.For example:uid or cn.
|
string
|
1.2.0
kyuubi\.authentication
\.ldap\.url|
<undefined>
|
SPACE character separated LDAP connection URL(s).
|
string
|
1.0.0
+kyuubi\.authentication
\.sasl\.enabled|
true
|
Whether enable SASL(GSSAPI or PLAIN) mechanism for authentication
|
boolean
|
1.4.0
+kyuubi\.authentication
\.sasl\.kerberos\.enabled|
false
|
Whether enable KERBEROS: Kerberos/GSSAPI authentication
|
boolean
|
1.4.0
+kyuubi\.authentication
\.sasl\.plain\.auth\.type|
<undefined>
|
Client authentication types for PLAIN mechanism.
|
string
|
1.4.0
kyuubi\.authentication
\.sasl\.qop|
auth
|
Sasl QOP enable higher levels of protection for Kyuubi communication with clients.
|
string
|
1.0.0
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 0d04e84aa28..936d4d947d9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -26,7 +26,7 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.engine.ShareLevel -import org.apache.kyuubi.service.authentication.{AuthTypes, SaslQOP} +import org.apache.kyuubi.service.authentication.{PlainAuthTypes, SaslQOP} case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { import KyuubiConf._ @@ -136,7 +136,9 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { FRONTEND_THRIFT_BINARY_BIND_PORT, FRONTEND_REST_BIND_HOST, FRONTEND_REST_BIND_PORT, - AUTHENTICATION_METHOD, + AUTHENTICATION_SASL_ENABLED, + AUTHENTICATION_SASL_KERBEROS_ENABLED, + AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, SERVER_KEYTAB, SERVER_PRINCIPAL, KINIT_INTERVAL) @@ -372,18 +374,30 @@ object KyuubiConf { .version("1.4.0") .fallbackConf(FRONTEND_LOGIN_BACKOFF_SLOT_LENGTH) - val AUTHENTICATION_METHOD: ConfigEntry[String] = buildConf("authentication") - .doc("Client authentication types.") - .version("1.0.0") - .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValues(AuthTypes.values.map(_.toString)) - .createWithDefault(AuthTypes.NONE.toString) + val AUTHENTICATION_SASL_ENABLED: ConfigEntry[Boolean] = buildConf("authentication.sasl.enabled") + .doc("Whether enable SASL(GSSAPI or PLAIN) mechanism for authentication") + .version("1.4.0") + .booleanConf + .createWithDefault(true) + + val AUTHENTICATION_SASL_KERBEROS_ENABLED: ConfigEntry[Boolean] = + buildConf("authentication.sasl.kerberos.enabled") + .doc("Whether enable KERBEROS: Kerberos/GSSAPI authentication") + .version("1.4.0") + .booleanConf + .createWithDefault(false) + + val AUTHENTICATION_SASL_PLAIN_AUTH_TYPE: OptionalConfigEntry[String] = + buildConf("authentication.sasl.plain.auth.type") + .doc("Client authentication types for PLAIN mechanism.") + .version("1.4.0") + .stringConf + .transform(_.toUpperCase(Locale.ROOT)) + .checkValues(PlainAuthTypes.values.map(_.toString)) + .createOptional val AUTHENTICATION_CUSTOM_CLASS: OptionalConfigEntry[String] = buildConf("authentication.custom.class") diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index 736b6e7f269..b8b2e1ad5c9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -32,14 +32,16 @@ import org.apache.thrift.transport.{TTransportException, TTransportFactory} import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ -import org.apache.kyuubi.service.authentication.AuthTypes._ +import org.apache.kyuubi.service.authentication.PlainAuthTypes._ class KyuubiAuthenticationFactory(conf: KyuubiConf) { + private val saslEnabled: Boolean = conf.get(AUTHENTICATION_SASL_ENABLED) + private val kerberosEnabled: Boolean = conf.get(AUTHENTICATION_SASL_KERBEROS_ENABLED) + private val plainAuthType: Option[PlainAuthType] = + conf.get(AUTHENTICATION_SASL_PLAIN_AUTH_TYPE).map(PlainAuthTypes.withName) - private val authType: AuthType = AuthTypes.withName(conf.get(AUTHENTICATION_METHOD)) - - private val saslServer: Option[HadoopThriftAuthBridgeServer] = authType match { - case KERBEROS => + private val hadoopAuthServer: Option[HadoopThriftAuthBridgeServer] = { + if (saslEnabled && kerberosEnabled) { val secretMgr = KyuubiDelegationTokenManager(conf) try { secretMgr.startThreads() @@ -47,7 +49,9 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { case e: IOException => throw new TTransportException("Failed to start token manager", e) } Some(new HadoopThriftAuthBridgeServer(secretMgr)) - case _ => None + } else { + None + } } private def getSaslProperties: java.util.Map[String, String] = { @@ -59,34 +63,40 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { } def getTTransportFactory: TTransportFactory = { - saslServer match { - case Some(server) => - val serverTransportFactory = try { - server.createSaslServerTransportFactory(getSaslProperties) - } catch { - case e: TTransportException => throw new LoginException(e.getMessage) - } + if (!saslEnabled || (!kerberosEnabled && plainAuthType.isEmpty)) { + new TTransportFactory() + } else { + val kerberosTransportFactory = hadoopAuthServer match { + case Some(server) => + val transportFactory = try { + server.createSaslServerTransportFactory(getSaslProperties) + } catch { + case e: TTransportException => throw new LoginException(e.getMessage) + } + Some(transportFactory) + + case _ => None + } - server.wrapTransportFactory(serverTransportFactory) + val transportFactory = plainAuthType.map { authType => + PlainSASLHelper.getTransportFactory(authType.toString, conf, kerberosTransportFactory) + }.orElse(kerberosTransportFactory).orNull - case _ => authType match { - case NOSASL => new TTransportFactory - case _ => PlainSASLHelper.getTransportFactory(authType.toString, conf) - } + hadoopAuthServer.map(_.wrapTransportFactory(transportFactory)).getOrElse(transportFactory) } } - def getTProcessorFactory(fe: Iface): TProcessorFactory = saslServer match { + def getTProcessorFactory(fe: Iface): TProcessorFactory = hadoopAuthServer match { case Some(server) => FEServiceProcessorFactory(server, fe) case _ => PlainSASLHelper.getProcessFactory(fe) } def getRemoteUser: Option[String] = { - saslServer.map(_.getRemoteUser).orElse(Option(TSetIpAddressProcessor.getUserName)) + hadoopAuthServer.map(_.getRemoteUser).orElse(Option(TSetIpAddressProcessor.getUserName)) } def getIpAddress: Option[String] = { - saslServer.map(_.getRemoteAddress).map(_.getHostAddress) + hadoopAuthServer.map(_.getRemoteAddress).map(_.getHostAddress) .orElse(Option(TSetIpAddressProcessor.getUserIpAddress)) } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainAuthTypes.scala similarity index 88% rename from kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala rename to kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainAuthTypes.scala index eacebad31ca..81af956ecd3 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainAuthTypes.scala @@ -17,8 +17,8 @@ package org.apache.kyuubi.service.authentication -object AuthTypes extends Enumeration { - type AuthType = Value +object PlainAuthTypes extends Enumeration { + type PlainAuthType = Value - val NOSASL, NONE, LDAP, KERBEROS, CUSTOM = Value + val NONE, LDAP, CUSTOM = Value } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLHelper.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLHelper.scala index 07cc501dcf5..2c597ec2215 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLHelper.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLHelper.scala @@ -71,8 +71,11 @@ object PlainSASLHelper { SQLPlainProcessorFactory(service) } - def getTransportFactory(authTypeStr: String, conf: KyuubiConf): TTransportFactory = { - val saslFactory = new TSaslServerTransport.Factory() + def getTransportFactory( + authTypeStr: String, + conf: KyuubiConf, + transportFactory: Option[TSaslServerTransport.Factory] = None): TTransportFactory = { + val saslFactory = transportFactory.getOrElse(new TSaslServerTransport.Factory()) try { val handler = new PlainServerCallbackHandler(authTypeStr, conf) val props = new java.util.HashMap[String, String] diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala index fe990911890..3871f82d6c8 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala @@ -39,6 +39,7 @@ class ThriftFrontendServiceSuite extends KyuubiFunSuite { protected val conf = KyuubiConf() .set(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) .set("kyuubi.test.server.should.fail", "false") + .set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") val user: String = System.getProperty("user.name") val sessionConf: util.Map[String, String] = new util.HashMap() diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index 50aa53638ef..0d0f480fb6b 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -25,7 +25,6 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider import org.apache.kyuubi.util.KyuubiHadoopUtils - class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { import KyuubiAuthenticationFactory._ @@ -45,7 +44,7 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { } test("AuthType NONE") { - val kyuubiConf = KyuubiConf() + val kyuubiConf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") val auth = new KyuubiAuthenticationFactory(kyuubiConf) auth.getTTransportFactory assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) @@ -55,14 +54,14 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { } test("AuthType Other") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, "INVALID") + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "INVALID") val e = intercept[IllegalArgumentException](new KyuubiAuthenticationFactory(conf)) - assert(e.getMessage === "The value of kyuubi.authentication should be one of" + - " CUSTOM, KERBEROS, LDAP, NONE, NOSASL, but was INVALID") + assert(e.getMessage === "The value of kyuubi.authentication.sasl.plain.auth.type should be" + + " one of CUSTOM, LDAP, NONE, but was INVALID") } test("AuthType LDAP") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, "LDAP") + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "LDAP") val authFactory = new KyuubiAuthenticationFactory(conf) authFactory.getTTransportFactory assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) @@ -70,7 +69,7 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { test("AuthType KERBEROS w/o keytab/principal") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, "KERBEROS") + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_KERBEROS_ENABLED, true) val factory = new KyuubiAuthenticationFactory(conf) val e = intercept[LoginException](factory.getTTransportFactory) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala index 95432d0562b..37856cfc1f3 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala @@ -37,6 +37,7 @@ class SessionManagerSuite extends ThriftFrontendServiceSuite with Eventually { .set(KyuubiConf.OPERATION_IDLE_TIMEOUT, Duration.ofSeconds(20).toMillis) .set(KyuubiConf.SESSION_CONF_RESTRICT_LIST, Seq("spark.*")) .set(KyuubiConf.SESSION_CONF_IGNORE_LIST, Seq("session.engine.*")) + .set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") test("close expired operations") { withSessionHandle{ (client, handle) => diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala index 06335f1812b..496eb88122b 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala @@ -283,9 +283,15 @@ object ServiceDiscovery extends Logging { confsToPublish += ("hive.server2.thrift.port" -> hostPort(1)) confsToPublish += ("hive.server2.thrift.sasl.qop" -> conf.get(KyuubiConf.SASL_QOP)) // Auth specific confs - val authenticationMethod = conf.get(KyuubiConf.AUTHENTICATION_METHOD) - confsToPublish += ("hive.server2.authentication" -> authenticationMethod) - if (authenticationMethod.equalsIgnoreCase("KERBEROS")) { + confsToPublish += ("hive.server2.authentication.sasl.enabled" -> + conf.get(KyuubiConf.AUTHENTICATION_SASL_ENABLED).toString) + confsToPublish += ("hive.server2.authentication.sasl.kerberos.enabled" -> + conf.get(KyuubiConf.AUTHENTICATION_SASL_KERBEROS_ENABLED).toString) + conf.get(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE).foreach { plainAuthType => + confsToPublish += ("hive.server2.authentication.sasl.plain.auth.type" -> plainAuthType) + } + if (conf.get(KyuubiConf.AUTHENTICATION_SASL_ENABLED) && + conf.get(KyuubiConf.AUTHENTICATION_SASL_KERBEROS_ENABLED)) { confsToPublish += ("hive.server2.authentication.kerberos.principal" -> conf.get(KyuubiConf.SERVER_PRINCIPAL).map(KyuubiHadoopUtils.getServerPrincipal) .getOrElse("")) From 314579f1fa70d451afc960abab958e25f7e7fe4a Mon Sep 17 00:00:00 2001 From: fwang12 Date: Wed, 20 Oct 2021 19:09:51 +0800 Subject: [PATCH 02/27] update default --- .../org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala | 1 - .../authentication/KyuubiAuthenticationFactorySuite.scala | 2 +- .../scala/org/apache/kyuubi/session/SessionManagerSuite.scala | 1 - pom.xml | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala index 3871f82d6c8..fe990911890 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/ThriftFrontendServiceSuite.scala @@ -39,7 +39,6 @@ class ThriftFrontendServiceSuite extends KyuubiFunSuite { protected val conf = KyuubiConf() .set(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) .set("kyuubi.test.server.should.fail", "false") - .set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") val user: String = System.getProperty("user.name") val sessionConf: util.Map[String, String] = new util.HashMap() diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index 0d0f480fb6b..848d4604a68 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -44,7 +44,7 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { } test("AuthType NONE") { - val kyuubiConf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") + val kyuubiConf = KyuubiConf() val auth = new KyuubiAuthenticationFactory(kyuubiConf) auth.getTTransportFactory assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala index 37856cfc1f3..95432d0562b 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/session/SessionManagerSuite.scala @@ -37,7 +37,6 @@ class SessionManagerSuite extends ThriftFrontendServiceSuite with Eventually { .set(KyuubiConf.OPERATION_IDLE_TIMEOUT, Duration.ofSeconds(20).toMillis) .set(KyuubiConf.SESSION_CONF_RESTRICT_LIST, Seq("spark.*")) .set(KyuubiConf.SESSION_CONF_IGNORE_LIST, Seq("session.engine.*")) - .set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") test("close expired operations") { withSessionHandle{ (client, handle) => diff --git a/pom.xml b/pom.xml index 88d8f1ce892..7805aacd70c 100644 --- a/pom.xml +++ b/pom.xml @@ -1428,6 +1428,7 @@ false target/server_operation_logs target/engine_operation_logs + NONE ${maven.plugin.scalatest.exclude.tags} ${maven.plugin.scalatest.include.tags} From bb75f8e9091fd2454da0c4d0e45bc80f361a2a84 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Wed, 20 Oct 2021 21:11:12 +0800 Subject: [PATCH 03/27] save --- .../authentication/KyuubiAuthenticationFactory.scala | 6 ++++-- .../org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index b8b2e1ad5c9..b587d22c7a6 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -73,16 +73,18 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { } catch { case e: TTransportException => throw new LoginException(e.getMessage) } + server.wrapTransportFactory(transportFactory) Some(transportFactory) case _ => None } val transportFactory = plainAuthType.map { authType => - PlainSASLHelper.getTransportFactory(authType.toString, conf, kerberosTransportFactory) + PlainSASLHelper.getTransportFactory(authType.toString, conf) }.orElse(kerberosTransportFactory).orNull - hadoopAuthServer.map(_.wrapTransportFactory(transportFactory)).getOrElse(transportFactory) + // hadoopAuthServer.map(_.wrapTransportFactory(transportFactory)).getOrElse(transportFactory) + transportFactory } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala index bbc27a71007..233971541e7 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala @@ -31,6 +31,7 @@ class DDLTPCDSSuite extends WithKyuubiServer override protected val conf: KyuubiConf = { val kyuubiConf = KyuubiConf().set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 20000L) + .set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE.key, "NONE") extraConfigs.foreach { case (k, v) => kyuubiConf.set(k, v) } kyuubiConf } From 98f936405cf27f0c0ebda059ab70268ab57ee057 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 21 Oct 2021 18:19:23 +0800 Subject: [PATCH 04/27] revert sth --- .../authentication/KyuubiAuthenticationFactory.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index b587d22c7a6..b8b2e1ad5c9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -73,18 +73,16 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { } catch { case e: TTransportException => throw new LoginException(e.getMessage) } - server.wrapTransportFactory(transportFactory) Some(transportFactory) case _ => None } val transportFactory = plainAuthType.map { authType => - PlainSASLHelper.getTransportFactory(authType.toString, conf) + PlainSASLHelper.getTransportFactory(authType.toString, conf, kerberosTransportFactory) }.orElse(kerberosTransportFactory).orNull - // hadoopAuthServer.map(_.wrapTransportFactory(transportFactory)).getOrElse(transportFactory) - transportFactory + hadoopAuthServer.map(_.wrapTransportFactory(transportFactory)).getOrElse(transportFactory) } } From 10f788ae22ad005b826d47edd0bebd68fe0e147b Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 21 Oct 2021 18:28:13 +0800 Subject: [PATCH 05/27] before all --- .../src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala | 3 +++ .../org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala index 35d328e01b2..881c32aa847 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi +import org.apache.kyuubi.config.KyuubiConf + import scala.collection.mutable.ArrayBuffer // scalastyle:off @@ -43,6 +45,7 @@ trait KyuubiFunSuite extends AnyFunSuite // scalastyle:on override def beforeAll(): Unit = { System.setProperty(IS_TESTING.key, "true") + System.setProperty(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE.key, "NONE") doThreadPreAudit() super.beforeAll() } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala index 233971541e7..bbc27a71007 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/tpcds/DDLTPCDSSuite.scala @@ -31,7 +31,6 @@ class DDLTPCDSSuite extends WithKyuubiServer override protected val conf: KyuubiConf = { val kyuubiConf = KyuubiConf().set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 20000L) - .set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE.key, "NONE") extraConfigs.foreach { case (k, v) => kyuubiConf.set(k, v) } kyuubiConf } From 8cc2ea6694c2ad0c105a1167e8e927fafdf94855 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 21 Oct 2021 21:11:31 +0800 Subject: [PATCH 06/27] fix ut --- .../src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala | 3 --- .../src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala index 881c32aa847..35d328e01b2 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala @@ -17,8 +17,6 @@ package org.apache.kyuubi -import org.apache.kyuubi.config.KyuubiConf - import scala.collection.mutable.ArrayBuffer // scalastyle:off @@ -45,7 +43,6 @@ trait KyuubiFunSuite extends AnyFunSuite // scalastyle:on override def beforeAll(): Unit = { System.setProperty(IS_TESTING.key, "true") - System.setProperty(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE.key, "NONE") doThreadPreAudit() super.beforeAll() } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala index cc6d45cc8e9..398bb89940d 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala @@ -45,6 +45,7 @@ trait WithKyuubiServer extends KyuubiFunSuite { conf.setIfMissing(ENGINE_IDLE_TIMEOUT, 10000L) conf.set(HA_ZK_QUORUM, zkServer.getConnectString) conf.set(HA_ZK_ACL_ENABLED, false) + conf.set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") // TODO KYUUBI #745 conf.setIfMissing(ENGINE_INIT_TIMEOUT, 300000L) From 1aa30a5cafa02c4864dc9c7ff487b1426713b8f5 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Thu, 21 Oct 2021 22:14:13 +0800 Subject: [PATCH 07/27] refactor --- docs/deployment/settings.md | 4 +- .../org/apache/kyuubi/config/KyuubiConf.scala | 44 +++++++------------ .../{PlainAuthTypes.scala => AuthTypes.scala} | 6 +-- .../KyuubiAuthenticationFactory.scala | 39 +++++++++------- .../KyuubiAuthenticationFactorySuite.scala | 10 ++--- .../kyuubi/ha/client/ServiceDiscovery.scala | 12 ++--- .../org/apache/kyuubi/WithKyuubiServer.scala | 1 - pom.xml | 1 - 8 files changed, 51 insertions(+), 66 deletions(-) rename kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/{PlainAuthTypes.scala => AuthTypes.scala} (88%) diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 7bb1aa6b8f1..204393631dc 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -122,14 +122,12 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- +kyuubi\.authentication|
NONE
|
A comma separated list of client authentication types.
  • NOSASL: raw transport.
  • NONE: no authentication check.
  • KERBEROS: Kerberos/GSSAPI authentication.
  • CUSTOM: User-defined authentication.
  • LDAP: Lightweight Directory Access Protocol authentication.
|
seq
|
1.0.0
kyuubi\.authentication
\.custom\.class|
<undefined>
|
User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider
|
string
|
1.3.0
kyuubi\.authentication
\.ldap\.base\.dn|
<undefined>
|
LDAP base DN.
|
string
|
1.0.0
kyuubi\.authentication
\.ldap\.domain|
<undefined>
|
LDAP domain.
|
string
|
1.0.0
kyuubi\.authentication
\.ldap\.guidKey|
uid
|
LDAP attribute name whose values are unique in this LDAP server.For example:uid or cn.
|
string
|
1.2.0
kyuubi\.authentication
\.ldap\.url|
<undefined>
|
SPACE character separated LDAP connection URL(s).
|
string
|
1.0.0
-kyuubi\.authentication
\.sasl\.enabled|
true
|
Whether enable SASL(GSSAPI or PLAIN) mechanism for authentication
|
boolean
|
1.4.0
-kyuubi\.authentication
\.sasl\.kerberos\.enabled|
false
|
Whether enable KERBEROS: Kerberos/GSSAPI authentication
|
boolean
|
1.4.0
-kyuubi\.authentication
\.sasl\.plain\.auth\.type|
<undefined>
|
Client authentication types for PLAIN mechanism.
  • NONE: no authentication check.
  • CUSTOM: User-defined authentication.
  • LDAP: Lightweight Directory Access Protocol authentication.
|
string
|
1.4.0
kyuubi\.authentication
\.sasl\.qop|
auth
|
Sasl QOP enable higher levels of protection for Kyuubi communication with clients.
  • auth - authentication only (default)
  • auth-int - authentication plus integrity protection
  • auth-conf - authentication plus integrity and confidentiality protection. This is applicable only if Kyuubi is configured to use Kerberos authentication.
|
string
|
1.0.0
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 936d4d947d9..2d0f261168a 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -26,7 +26,7 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.engine.ShareLevel -import org.apache.kyuubi.service.authentication.{PlainAuthTypes, SaslQOP} +import org.apache.kyuubi.service.authentication.{AuthTypes, SaslQOP} case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { import KyuubiConf._ @@ -136,9 +136,7 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { FRONTEND_THRIFT_BINARY_BIND_PORT, FRONTEND_REST_BIND_HOST, FRONTEND_REST_BIND_PORT, - AUTHENTICATION_SASL_ENABLED, - AUTHENTICATION_SASL_KERBEROS_ENABLED, - AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, + AUTHENTICATION_METHOD, SERVER_KEYTAB, SERVER_PRINCIPAL, KINIT_INTERVAL) @@ -374,30 +372,20 @@ object KyuubiConf { .version("1.4.0") .fallbackConf(FRONTEND_LOGIN_BACKOFF_SLOT_LENGTH) - val AUTHENTICATION_SASL_ENABLED: ConfigEntry[Boolean] = buildConf("authentication.sasl.enabled") - .doc("Whether enable SASL(GSSAPI or PLAIN) mechanism for authentication") - .version("1.4.0") - .booleanConf - .createWithDefault(true) - - val AUTHENTICATION_SASL_KERBEROS_ENABLED: ConfigEntry[Boolean] = - buildConf("authentication.sasl.kerberos.enabled") - .doc("Whether enable KERBEROS: Kerberos/GSSAPI authentication") - .version("1.4.0") - .booleanConf - .createWithDefault(false) - - val AUTHENTICATION_SASL_PLAIN_AUTH_TYPE: OptionalConfigEntry[String] = - buildConf("authentication.sasl.plain.auth.type") - .doc("Client authentication types for PLAIN mechanism.
    " + - "
  • NONE: no authentication check.
    • " + - "
  • CUSTOM: User-defined authentication.
    • " + - "
  • LDAP: Lightweight Directory Access Protocol authentication.
") - .version("1.4.0") - .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValues(PlainAuthTypes.values.map(_.toString)) - .createOptional + val AUTHENTICATION_METHOD: ConfigEntry[Seq[String]] = buildConf("authentication") + .doc("A comma separated list of client authentication types.
    " + + "
  • NOSASL: raw transport.
    • " + + "
  • NONE: no authentication check.
    • " + + "
  • KERBEROS: Kerberos/GSSAPI authentication.
    • " + + "
  • CUSTOM: User-defined authentication.
    • " + + "
  • LDAP: Lightweight Directory Access Protocol authentication.
") + .version("1.0.0") + .stringConf + .toSequence() + .transform(_.map(_.toUpperCase(Locale.ROOT))) + .checkValue(_.forall(AuthTypes.values.map(_.toString).contains), + s"the authentication type should be one or more of ${AuthTypes.values.mkString(",")}") + .createWithDefault(Seq(AuthTypes.NONE.toString)) val AUTHENTICATION_CUSTOM_CLASS: OptionalConfigEntry[String] = buildConf("authentication.custom.class") diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainAuthTypes.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala similarity index 88% rename from kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainAuthTypes.scala rename to kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala index 81af956ecd3..eacebad31ca 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainAuthTypes.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/AuthTypes.scala @@ -17,8 +17,8 @@ package org.apache.kyuubi.service.authentication -object PlainAuthTypes extends Enumeration { - type PlainAuthType = Value +object AuthTypes extends Enumeration { + type AuthType = Value - val NONE, LDAP, CUSTOM = Value + val NOSASL, NONE, LDAP, KERBEROS, CUSTOM = Value } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index b8b2e1ad5c9..1098a9225ca 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -27,21 +27,22 @@ import org.apache.hadoop.security.authentication.util.KerberosName import org.apache.hadoop.security.authorize.ProxyUsers import org.apache.hive.service.rpc.thrift.TCLIService.Iface import org.apache.thrift.TProcessorFactory -import org.apache.thrift.transport.{TTransportException, TTransportFactory} +import org.apache.thrift.transport.{TSaslServerTransport, TTransportException, TTransportFactory} import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ -import org.apache.kyuubi.service.authentication.PlainAuthTypes._ +import org.apache.kyuubi.service.authentication.AuthTypes._ class KyuubiAuthenticationFactory(conf: KyuubiConf) { - private val saslEnabled: Boolean = conf.get(AUTHENTICATION_SASL_ENABLED) - private val kerberosEnabled: Boolean = conf.get(AUTHENTICATION_SASL_KERBEROS_ENABLED) - private val plainAuthType: Option[PlainAuthType] = - conf.get(AUTHENTICATION_SASL_PLAIN_AUTH_TYPE).map(PlainAuthTypes.withName) + + private val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName).toSet match { + case s if s == Set(NOSASL) => s + case s => s.filterNot(_.equals(NOSASL)) + } private val hadoopAuthServer: Option[HadoopThriftAuthBridgeServer] = { - if (saslEnabled && kerberosEnabled) { + if (authTypes.contains(KERBEROS)) { val secretMgr = KyuubiDelegationTokenManager(conf) try { secretMgr.startThreads() @@ -63,26 +64,32 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { } def getTTransportFactory: TTransportFactory = { - if (!saslEnabled || (!kerberosEnabled && plainAuthType.isEmpty)) { + if (authTypes == Set(NOSASL)) { new TTransportFactory() } else { - val kerberosTransportFactory = hadoopAuthServer match { + var transportFactory: TSaslServerTransport.Factory = null + + hadoopAuthServer match { case Some(server) => - val transportFactory = try { + transportFactory = try { server.createSaslServerTransportFactory(getSaslProperties) } catch { case e: TTransportException => throw new LoginException(e.getMessage) } - Some(transportFactory) - case _ => None + case _ => } - val transportFactory = plainAuthType.map { authType => - PlainSASLHelper.getTransportFactory(authType.toString, conf, kerberosTransportFactory) - }.orElse(kerberosTransportFactory).orNull + authTypes.filterNot(at => at.equals(KERBEROS)).foreach { plainAuthType => + transportFactory = PlainSASLHelper.getTransportFactory(plainAuthType.toString, conf, + Option(transportFactory)).asInstanceOf[TSaslServerTransport.Factory] + } + + hadoopAuthServer match { + case Some(server) => server.wrapTransportFactory(transportFactory) - hadoopAuthServer.map(_.wrapTransportFactory(transportFactory)).getOrElse(transportFactory) + case _ => transportFactory + } } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index 848d4604a68..bdee687a430 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -54,14 +54,14 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { } test("AuthType Other") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "INVALID") + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("INVALID")) val e = intercept[IllegalArgumentException](new KyuubiAuthenticationFactory(conf)) - assert(e.getMessage === "The value of kyuubi.authentication.sasl.plain.auth.type should be" + - " one of CUSTOM, LDAP, NONE, but was INVALID") + assert(e.getMessage === "the authentication type should be one or more of" + + " NOSASL,NONE,LDAP,KERBEROS,CUSTOM") } test("AuthType LDAP") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "LDAP") + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP")) val authFactory = new KyuubiAuthenticationFactory(conf) authFactory.getTTransportFactory assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) @@ -69,7 +69,7 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { test("AuthType KERBEROS w/o keytab/principal") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_SASL_KERBEROS_ENABLED, true) + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS")) val factory = new KyuubiAuthenticationFactory(conf) val e = intercept[LoginException](factory.getTTransportFactory) diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala index 496eb88122b..6f7483a5891 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala @@ -283,15 +283,9 @@ object ServiceDiscovery extends Logging { confsToPublish += ("hive.server2.thrift.port" -> hostPort(1)) confsToPublish += ("hive.server2.thrift.sasl.qop" -> conf.get(KyuubiConf.SASL_QOP)) // Auth specific confs - confsToPublish += ("hive.server2.authentication.sasl.enabled" -> - conf.get(KyuubiConf.AUTHENTICATION_SASL_ENABLED).toString) - confsToPublish += ("hive.server2.authentication.sasl.kerberos.enabled" -> - conf.get(KyuubiConf.AUTHENTICATION_SASL_KERBEROS_ENABLED).toString) - conf.get(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE).foreach { plainAuthType => - confsToPublish += ("hive.server2.authentication.sasl.plain.auth.type" -> plainAuthType) - } - if (conf.get(KyuubiConf.AUTHENTICATION_SASL_ENABLED) && - conf.get(KyuubiConf.AUTHENTICATION_SASL_KERBEROS_ENABLED)) { + val authenticationMethod = conf.get(KyuubiConf.AUTHENTICATION_METHOD).mkString(",") + confsToPublish += ("hive.server2.authentication" -> authenticationMethod) + if (authenticationMethod.equalsIgnoreCase("KERBEROS")) { confsToPublish += ("hive.server2.authentication.kerberos.principal" -> conf.get(KyuubiConf.SERVER_PRINCIPAL).map(KyuubiHadoopUtils.getServerPrincipal) .getOrElse("")) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala index 398bb89940d..cc6d45cc8e9 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithKyuubiServer.scala @@ -45,7 +45,6 @@ trait WithKyuubiServer extends KyuubiFunSuite { conf.setIfMissing(ENGINE_IDLE_TIMEOUT, 10000L) conf.set(HA_ZK_QUORUM, zkServer.getConnectString) conf.set(HA_ZK_ACL_ENABLED, false) - conf.set(KyuubiConf.AUTHENTICATION_SASL_PLAIN_AUTH_TYPE, "NONE") // TODO KYUUBI #745 conf.setIfMissing(ENGINE_INIT_TIMEOUT, 300000L) diff --git a/pom.xml b/pom.xml index 7805aacd70c..88d8f1ce892 100644 --- a/pom.xml +++ b/pom.xml @@ -1428,7 +1428,6 @@ false target/server_operation_logs target/engine_operation_logs - NONE ${maven.plugin.scalatest.exclude.tags} ${maven.plugin.scalatest.include.tags} From 8545a03352e02b78ee810d01baa774a81a231ed8 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 16:29:04 +0800 Subject: [PATCH 08/27] add ut --- .../CustomAuthenticationProviderImplSuite.scala | 2 +- .../KyuubiAuthenticationFactorySuite.scala | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala index cfd4743d7e4..c5561d044fb 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala @@ -33,7 +33,7 @@ class CustomAuthenticationProviderImplSuite extends KyuubiFunSuite { "authentication.custom.class must be set when auth method was CUSTOM.")) conf.set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, - "org.apache.kyuubi.service.authentication.UserDefineAuthenticationProviderImpl") + classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) val p1 = getAuthenticationProvider(AuthMethods.withName("CUSTOM"), conf) val e2 = intercept[AuthenticationException](p1.authenticate("test", "test")) assert(e2.getMessage.contains("Username or password is not valid!")) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index bdee687a430..fbd22c023eb 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -75,4 +75,14 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { val e = intercept[LoginException](factory.getTTransportFactory) assert(e.getMessage startsWith "Kerberos principal should have 3 parts") } + + test("Support multiple kinds of authentication type") { + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NONE", "CUSTOM")) + .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, + classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) + + val authFactory = new KyuubiAuthenticationFactory(conf) + authFactory.getTTransportFactory + assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) + } } From 8dadfd322e2905e1164c5b5869cbac3f9c78b486 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 19:42:40 +0800 Subject: [PATCH 09/27] refactor ldap suite --- .../LdapAuthenticationProviderImplSuite.scala | 18 ++------ .../authentication/WithLdapServer.scala | 43 +++++++++++++++++++ 2 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala index 98d20690e03..474e15cb797 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala @@ -20,28 +20,18 @@ package org.apache.kyuubi.service.authentication import javax.naming.CommunicationException import javax.security.sasl.AuthenticationException -import com.unboundid.ldap.listener.InMemoryDirectoryServer -import com.unboundid.ldap.listener.InMemoryDirectoryServerConfig - -import org.apache.kyuubi.KyuubiFunSuite import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ -class LdapAuthenticationProviderImplSuite extends KyuubiFunSuite { - - private var ldapServer: InMemoryDirectoryServer = _ +class LdapAuthenticationProviderImplSuite extends WithLdapServer { + override protected val user: String = "kentyao" + override protected val password: String = "kentyao" private val conf = new KyuubiConf() override def beforeAll(): Unit = { - val config = new InMemoryDirectoryServerConfig("ou=users") - config.addAdditionalBindCredentials("uid=kentyao,ou=users", "kentyao") - ldapServer = new InMemoryDirectoryServer(config) - ldapServer.startListening() - - conf.set(AUTHENTICATION_LDAP_URL, s"ldap://localhost:" + ldapServer.getListenPort) - super.beforeAll() + conf.set(AUTHENTICATION_LDAP_URL, ldapUrl) } override def afterAll(): Unit = { diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala new file mode 100644 index 00000000000..d581c8ec046 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication + +import com.unboundid.ldap.listener.{InMemoryDirectoryServer, InMemoryDirectoryServerConfig} + +import org.apache.kyuubi.{KyuubiFunSuite, Utils} + +class WithLdapServer extends KyuubiFunSuite { + protected var ldapServer: InMemoryDirectoryServer = _ + protected val user = Utils.currentUser + protected val password = "password" + + protected def ldapUrl = s"ldap://localhost:${ldapServer.getListenPort}" + + override def beforeAll(): Unit = { + val config = new InMemoryDirectoryServerConfig("ou=users") + config.addAdditionalBindCredentials(s"uid=$user,ou=users", password) + ldapServer = new InMemoryDirectoryServer(config) + ldapServer.startListening() + super.beforeAll() + } + + override def afterAll(): Unit = { + ldapServer.close() + super.afterAll() + } +} From 7dc7c927a4b71c623bc14734d0402b51bfd3290e Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 20:11:05 +0800 Subject: [PATCH 10/27] with password --- .../org/apache/kyuubi/operation/JDBCTestUtils.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala index 36b7e60b1ed..96cc46d1a94 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala @@ -82,9 +82,9 @@ trait JDBCTestUtils extends KyuubiFunSuite { intercept[SQLException](DriverManager.getConnection(jdbcUrl, user, "")) } - def withMultipleConnectionJdbcStatement( - tableNames: String*)(fs: (Statement => Unit)*): Unit = { - val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, "") } + def withMultipleConnectionJdbcStatementWithPasswd( + passwd: String)(tableNames: String*)(fs: (Statement => Unit)*): Unit = { + val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, passwd) } val statements = connections.map(_.createStatement()) try { @@ -106,6 +106,10 @@ trait JDBCTestUtils extends KyuubiFunSuite { } } + def withMultipleConnectionJdbcStatement(tableNames: String*)(fs: (Statement => Unit)*): Unit = { + withMultipleConnectionJdbcStatementWithPasswd("")( tableNames: _*)(fs: _*) + } + def withDatabases(dbNames: String*)(fs: (Statement => Unit)*): Unit = { val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, "") } val statements = connections.map(_.createStatement()) From e39e19e62f3699493283309d2acce2c1fe13b2f4 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 20:56:05 +0800 Subject: [PATCH 11/27] add it --- ...ustomAuthenticationProviderImplSuite.scala | 5 +- .../LdapAuthenticationProviderImplSuite.scala | 4 +- ...UserDefineAuthenticationProviderImpl.scala | 4 +- .../authentication/WithLdapServer.scala | 11 ++-- ...KyuubiOperationMultipleAuthTypeSuite.scala | 54 +++++++++++++++++++ 5 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala index c5561d044fb..d47a90ecf85 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala @@ -18,8 +18,7 @@ package org.apache.kyuubi.service.authentication import javax.security.sasl.AuthenticationException - -import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.{KyuubiFunSuite, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.AuthenticationProviderFactory.getAuthenticationProvider @@ -38,6 +37,6 @@ class CustomAuthenticationProviderImplSuite extends KyuubiFunSuite { val e2 = intercept[AuthenticationException](p1.authenticate("test", "test")) assert(e2.getMessage.contains("Username or password is not valid!")) - p1.authenticate("user", "password") + p1.authenticate(Utils.currentUser, "password") } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala index 474e15cb797..8adc2882a43 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala @@ -24,8 +24,8 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ class LdapAuthenticationProviderImplSuite extends WithLdapServer { - override protected val user: String = "kentyao" - override protected val password: String = "kentyao" + override protected val ldapUser: String = "kentyao" + override protected val ldapPasswd: String = "kentyao" private val conf = new KyuubiConf() diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala index 7a84103cc46..181d009e326 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala @@ -19,13 +19,13 @@ package org.apache.kyuubi.service.authentication import javax.security.sasl.AuthenticationException -import org.apache.kyuubi.Logging +import org.apache.kyuubi.{Logging, Utils} class UserDefineAuthenticationProviderImpl() extends PasswdAuthenticationProvider with Logging { override def authenticate(user: String, password: String): Unit = { - if (user == "user" && password == "password") { + if (user == Utils.currentUser && password == "password") { info(s"Success log in of user: $user") } else { throw new AuthenticationException("Username or password is not valid!") diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala index d581c8ec046..1fbfee00736 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala @@ -21,16 +21,17 @@ import com.unboundid.ldap.listener.{InMemoryDirectoryServer, InMemoryDirectorySe import org.apache.kyuubi.{KyuubiFunSuite, Utils} -class WithLdapServer extends KyuubiFunSuite { +trait WithLdapServer extends KyuubiFunSuite { protected var ldapServer: InMemoryDirectoryServer = _ - protected val user = Utils.currentUser - protected val password = "password" + protected val ldapBaseDn = "ou=users" + protected val ldapUser = Utils.currentUser + protected val ldapPasswd = "password" protected def ldapUrl = s"ldap://localhost:${ldapServer.getListenPort}" override def beforeAll(): Unit = { - val config = new InMemoryDirectoryServerConfig("ou=users") - config.addAdditionalBindCredentials(s"uid=$user,ou=users", password) + val config = new InMemoryDirectoryServerConfig(ldapBaseDn) + config.addAdditionalBindCredentials(s"uid=$ldapUser,ou=users", ldapPasswd) ldapServer = new InMemoryDirectoryServer(config) ldapServer.startListening() super.beforeAll() diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala new file mode 100644 index 00000000000..e56215b11b5 --- /dev/null +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.operation + +import org.apache.kyuubi.WithKyuubiServer +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.service.authentication.{UserDefineAuthenticationProviderImpl, WithLdapServer} + +class KyuubiOperationMultipleAuthTypeSuite extends + WithKyuubiServer with WithLdapServer with JDBCTestUtils { + override protected val ldapPasswd: String = "ldapPassword" + private val customPasswd: String = "password" + + override protected def jdbcUrl: String = getJdbcUrl + + override protected lazy val conf: KyuubiConf = { + KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM")) + .set(KyuubiConf.AUTHENTICATION_LDAP_URL, ldapUrl) + .set(KyuubiConf.AUTHENTICATION_LDAP_BASEDN, ldapBaseDn) + .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, + classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) + } + + test("test with LDAP authentication") { + withMultipleConnectionJdbcStatementWithPasswd(ldapPasswd)() { statement => + val resultSet = statement.executeQuery("select engine_name()") + assert(resultSet.next()) + assert(resultSet.getString(1).nonEmpty) + } + } + + test("test with CUSTOM authentication") { + withMultipleConnectionJdbcStatementWithPasswd(customPasswd)() { statement => + val resultSet = statement.executeQuery("select engine_name()") + assert(resultSet.next()) + assert(resultSet.getString(1).nonEmpty) + } + } +} From 7b590a237d76e7897139bcc321a3d5e7623dd42c Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 21:01:13 +0800 Subject: [PATCH 12/27] add ut --- .../LdapAuthenticationProviderImplSuite.scala | 2 +- .../service/authentication/WithLdapServer.scala | 4 ++-- .../KyuubiOperationMultipleAuthTypeSuite.scala | 14 ++++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala index 8adc2882a43..4c65b848d8c 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala @@ -25,7 +25,7 @@ import org.apache.kyuubi.config.KyuubiConf._ class LdapAuthenticationProviderImplSuite extends WithLdapServer { override protected val ldapUser: String = "kentyao" - override protected val ldapPasswd: String = "kentyao" + override protected val ldapUserPasswd: String = "kentyao" private val conf = new KyuubiConf() diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala index 1fbfee00736..ea1fa138751 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala @@ -25,13 +25,13 @@ trait WithLdapServer extends KyuubiFunSuite { protected var ldapServer: InMemoryDirectoryServer = _ protected val ldapBaseDn = "ou=users" protected val ldapUser = Utils.currentUser - protected val ldapPasswd = "password" + protected val ldapUserPasswd = "password" protected def ldapUrl = s"ldap://localhost:${ldapServer.getListenPort}" override def beforeAll(): Unit = { val config = new InMemoryDirectoryServerConfig(ldapBaseDn) - config.addAdditionalBindCredentials(s"uid=$ldapUser,ou=users", ldapPasswd) + config.addAdditionalBindCredentials(s"uid=$ldapUser,ou=users", ldapUserPasswd) ldapServer = new InMemoryDirectoryServer(config) ldapServer.startListening() super.beforeAll() diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala index e56215b11b5..f2809366299 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -17,13 +17,15 @@ package org.apache.kyuubi.operation +import java.sql.SQLException + import org.apache.kyuubi.WithKyuubiServer import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.{UserDefineAuthenticationProviderImpl, WithLdapServer} class KyuubiOperationMultipleAuthTypeSuite extends WithKyuubiServer with WithLdapServer with JDBCTestUtils { - override protected val ldapPasswd: String = "ldapPassword" + override protected val ldapUserPasswd: String = "ldapPassword" private val customPasswd: String = "password" override protected def jdbcUrl: String = getJdbcUrl @@ -37,7 +39,7 @@ class KyuubiOperationMultipleAuthTypeSuite extends } test("test with LDAP authentication") { - withMultipleConnectionJdbcStatementWithPasswd(ldapPasswd)() { statement => + withMultipleConnectionJdbcStatementWithPasswd(ldapUserPasswd)() { statement => val resultSet = statement.executeQuery("select engine_name()") assert(resultSet.next()) assert(resultSet.getString(1).nonEmpty) @@ -51,4 +53,12 @@ class KyuubiOperationMultipleAuthTypeSuite extends assert(resultSet.getString(1).nonEmpty) } } + + test("test with invalid password") { + intercept[SQLException] { + withMultipleConnectionJdbcStatementWithPasswd("falsePasswd")() { statement => + statement.executeQuery("select engine_name()") + } + } + } } From f4038e934c6516132bd9ca37bb0d3b73e1552f05 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 21:02:10 +0800 Subject: [PATCH 13/27] fix code style --- .../authentication/CustomAuthenticationProviderImplSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala index d47a90ecf85..64e599eb3c0 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala @@ -18,6 +18,7 @@ package org.apache.kyuubi.service.authentication import javax.security.sasl.AuthenticationException + import org.apache.kyuubi.{KyuubiFunSuite, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.AuthenticationProviderFactory.getAuthenticationProvider From dd706740e594ceafa521bd20df802db770caa3cd Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 21:45:19 +0800 Subject: [PATCH 14/27] retest --- .../authentication/LdapAuthenticationProviderImplSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala index 4c65b848d8c..2563dc00d10 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala @@ -57,8 +57,7 @@ class LdapAuthenticationProviderImplSuite extends WithLdapServer { assert(e3.getMessage.contains(user)) assert(e3.getCause.isInstanceOf[javax.naming.AuthenticationException]) - val dn = "ou=users" - conf.set(AUTHENTICATION_LDAP_BASEDN, dn) + conf.set(AUTHENTICATION_LDAP_BASEDN, ldapBaseDn) val providerImpl2 = new LdapAuthenticationProviderImpl(conf) providerImpl2.authenticate("kentyao", "kentyao") From cd813ecf761e62bf00c2ea3e3f16bf505ff8bbfa Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 22:37:12 +0800 Subject: [PATCH 15/27] refactor --- .../KyuubiAuthenticationFactory.scala | 27 +++++++------- .../kyuubi/operation/JDBCTestUtils.scala | 4 ++- .../KyuubiAuthenticationFactorySuite.scala | 11 +----- ...KyuubiOperationMultipleAuthTypeSuite.scala | 36 ++++++++++++------- 4 files changed, 43 insertions(+), 35 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index 1098a9225ca..390491a06f8 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -29,20 +29,21 @@ import org.apache.hive.service.rpc.thrift.TCLIService.Iface import org.apache.thrift.TProcessorFactory import org.apache.thrift.transport.{TSaslServerTransport, TTransportException, TTransportFactory} -import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.{KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ import org.apache.kyuubi.service.authentication.AuthTypes._ -class KyuubiAuthenticationFactory(conf: KyuubiConf) { +class KyuubiAuthenticationFactory(conf: KyuubiConf) extends Logging { - private val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName).toSet match { - case s if s == Set(NOSASL) => s - case s => s.filterNot(_.equals(NOSASL)) - } + private val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName) + private val noSasl = authTypes == Seq(NOSASL) + private val kerberosEnabled = authTypes.contains(KERBEROS) + private val plainAuthTypeOpt = authTypes.filterNot(_.equals(KERBEROS)) + .filterNot(_.equals(NOSASL)).headOption private val hadoopAuthServer: Option[HadoopThriftAuthBridgeServer] = { - if (authTypes.contains(KERBEROS)) { + if (kerberosEnabled) { val secretMgr = KyuubiDelegationTokenManager(conf) try { secretMgr.startThreads() @@ -64,7 +65,7 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { } def getTTransportFactory: TTransportFactory = { - if (authTypes == Set(NOSASL)) { + if (noSasl) { new TTransportFactory() } else { var transportFactory: TSaslServerTransport.Factory = null @@ -80,14 +81,16 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf) { case _ => } - authTypes.filterNot(at => at.equals(KERBEROS)).foreach { plainAuthType => - transportFactory = PlainSASLHelper.getTransportFactory(plainAuthType.toString, conf, - Option(transportFactory)).asInstanceOf[TSaslServerTransport.Factory] + plainAuthTypeOpt match { + case Some(plainAuthType) => + transportFactory = PlainSASLHelper.getTransportFactory(plainAuthType.toString, conf, + Option(transportFactory)).asInstanceOf[TSaslServerTransport.Factory] + + case _ => } hadoopAuthServer match { case Some(server) => server.wrapTransportFactory(transportFactory) - case _ => transportFactory } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala index 96cc46d1a94..d76840df12a 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala @@ -63,7 +63,9 @@ trait JDBCTestUtils extends KyuubiFunSuite { } } - private def jdbcUrlWithConf: String = { + protected def jdbcUrlWithConf(): String = jdbcUrlWithConf(jdbcUrl) + + protected def jdbcUrlWithConf(jdbcUrl: String): String = { val sessionConfStr = sessionConfigs.map(kv => kv._1 + "=" + kv._2).mkString(";") val sparkHiveConfStr = if (sparkHiveConfigs.isEmpty) { "" diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index fbd22c023eb..3032bf606cc 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -25,6 +25,7 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider import org.apache.kyuubi.util.KyuubiHadoopUtils + class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { import KyuubiAuthenticationFactory._ @@ -75,14 +76,4 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { val e = intercept[LoginException](factory.getTTransportFactory) assert(e.getMessage startsWith "Kerberos principal should have 3 parts") } - - test("Support multiple kinds of authentication type") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NONE", "CUSTOM")) - .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, - classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) - - val authFactory = new KyuubiAuthenticationFactory(conf) - authFactory.getTTransportFactory - assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) - } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala index f2809366299..c9b87529fce 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -17,47 +17,59 @@ package org.apache.kyuubi.operation -import java.sql.SQLException +import java.sql.{DriverManager, SQLException} -import org.apache.kyuubi.WithKyuubiServer +import org.apache.kyuubi.{KerberizedTestHelper, WithKyuubiServer} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.service.authentication.{UserDefineAuthenticationProviderImpl, WithLdapServer} +import org.apache.kyuubi.service.authentication.UserDefineAuthenticationProviderImpl class KyuubiOperationMultipleAuthTypeSuite extends - WithKyuubiServer with WithLdapServer with JDBCTestUtils { - override protected val ldapUserPasswd: String = "ldapPassword" + WithKyuubiServer with KerberizedTestHelper with JDBCTestUtils { private val customPasswd: String = "password" override protected def jdbcUrl: String = getJdbcUrl + private def kerberosJdbcUrl: String = jdbcUrl + s"principal=${testPrincipal}" override protected lazy val conf: KyuubiConf = { - KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP", "CUSTOM")) - .set(KyuubiConf.AUTHENTICATION_LDAP_URL, ldapUrl) - .set(KyuubiConf.AUTHENTICATION_LDAP_BASEDN, ldapBaseDn) + KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "CUSTOM")) + .set(KyuubiConf.SERVER_KEYTAB, testKeytab) + .set(KyuubiConf.SERVER_PRINCIPAL, testPrincipal) .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) } - test("test with LDAP authentication") { - withMultipleConnectionJdbcStatementWithPasswd(ldapUserPasswd)() { statement => + test("test with KERBEROS authentication") { + val conn = DriverManager.getConnection(jdbcUrlWithConf(kerberosJdbcUrl), user, "") + try { + val statement = conn.createStatement() val resultSet = statement.executeQuery("select engine_name()") assert(resultSet.next()) assert(resultSet.getString(1).nonEmpty) + } finally { + conn.close() } } test("test with CUSTOM authentication") { - withMultipleConnectionJdbcStatementWithPasswd(customPasswd)() { statement => + val conn = DriverManager.getConnection(jdbcUrlWithConf, user, customPasswd) + try { + val statement = conn.createStatement() val resultSet = statement.executeQuery("select engine_name()") assert(resultSet.next()) assert(resultSet.getString(1).nonEmpty) + } finally { + conn.close() } } test("test with invalid password") { intercept[SQLException] { - withMultipleConnectionJdbcStatementWithPasswd("falsePasswd")() { statement => + val conn = DriverManager.getConnection(jdbcUrlWithConf, user, "invalidPassword") + try { + val statement = conn.createStatement() statement.executeQuery("select engine_name()") + } finally { + conn.close() } } } From 6691cc5726fca53b56833f692167e4ce2887f648 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 22:37:28 +0800 Subject: [PATCH 16/27] save --- .../org/apache/kyuubi/operation/JDBCTestUtils.scala | 10 +++------- .../KyuubiAuthenticationFactorySuite.scala | 1 - .../KyuubiOperationMultipleAuthTypeSuite.scala | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala index d76840df12a..2cac5b9dd70 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala @@ -84,9 +84,9 @@ trait JDBCTestUtils extends KyuubiFunSuite { intercept[SQLException](DriverManager.getConnection(jdbcUrl, user, "")) } - def withMultipleConnectionJdbcStatementWithPasswd( - passwd: String)(tableNames: String*)(fs: (Statement => Unit)*): Unit = { - val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, passwd) } + def withMultipleConnectionJdbcStatement( + tableNames: String*)(fs: (Statement => Unit)*): Unit = { + val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, "") } val statements = connections.map(_.createStatement()) try { @@ -108,10 +108,6 @@ trait JDBCTestUtils extends KyuubiFunSuite { } } - def withMultipleConnectionJdbcStatement(tableNames: String*)(fs: (Statement => Unit)*): Unit = { - withMultipleConnectionJdbcStatementWithPasswd("")( tableNames: _*)(fs: _*) - } - def withDatabases(dbNames: String*)(fs: (Statement => Unit)*): Unit = { val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, "") } val statements = connections.map(_.createStatement()) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index 3032bf606cc..bdee687a430 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -25,7 +25,6 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider import org.apache.kyuubi.util.KyuubiHadoopUtils - class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { import KyuubiAuthenticationFactory._ diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala index c9b87529fce..16d9ed1a663 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -31,7 +31,7 @@ class KyuubiOperationMultipleAuthTypeSuite extends private def kerberosJdbcUrl: String = jdbcUrl + s"principal=${testPrincipal}" override protected lazy val conf: KyuubiConf = { - KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "CUSTOM")) + KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "CUSTOM", "LDAP")) .set(KyuubiConf.SERVER_KEYTAB, testKeytab) .set(KyuubiConf.SERVER_PRINCIPAL, testPrincipal) .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, From 4fc63081585cd3705c50bb6d11cdfde5a594c075 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sat, 23 Oct 2021 23:53:07 +0800 Subject: [PATCH 17/27] refactor kerbereos helper --- .../apache/kyuubi/KerberizedTestHelper.scala | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala index d5ba4a05c20..31ed23112cf 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala @@ -50,26 +50,33 @@ trait KerberizedTestHelper extends KyuubiFunSuite { private var kdc: MiniKdc = _ private var krb5ConfPath: String = _ - eventually(timeout(60.seconds), interval(1.second)) { - try { - kdc = new MiniKdc(kdcConf, baseDir) - kdc.start() - krb5ConfPath = kdc.getKrb5conf.getAbsolutePath - } catch { - case NonFatal(e) => - if (kdc != null) { - kdc.stop() - kdc = null - } - throw e - } - } - private val keytabFile = new File(baseDir, "kyuubi-test.keytab") protected val testKeytab: String = keytabFile.getAbsolutePath - protected var testPrincipal = s"client/$hostName" - kdc.createPrincipal(keytabFile, testPrincipal) - + protected var testPrincipal: String = _ + + override def beforeAll(): Unit = { + eventually(timeout(60.seconds), interval(1.second)) { + try { + kdc = new MiniKdc(kdcConf, baseDir) + kdc.start() + krb5ConfPath = kdc.getKrb5conf.getAbsolutePath + } catch { + case NonFatal(e) => + if (kdc != null) { + kdc.stop() + kdc = null + } + throw e + } + } + val tempTestPrincipal = s"client/$hostName" + kdc.createPrincipal(keytabFile, tempTestPrincipal) + rewriteKrb5Conf() + testPrincipal = tempTestPrincipal + "@" + kdc.getRealm + info(s"KerberizedTest Principal: $testPrincipal") + info(s"KerberizedTest Keytab: $testKeytab") + super.beforeAll() + } /** * Forked from Apache Spark @@ -113,13 +120,6 @@ trait KerberizedTestHelper extends KyuubiFunSuite { System.lineSeparator() + s" $key=$value" } - rewriteKrb5Conf() - - testPrincipal = testPrincipal + "@" + kdc.getRealm - - info(s"KerberizedTest Principal: $testPrincipal") - info(s"KerberizedTest Keytab: $testKeytab") - override def afterAll(): Unit = { kdc.stop() super.afterAll() From ee0e8bc060e43b27272e5a88597adf2713c3c0ff Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 00:33:27 +0800 Subject: [PATCH 18/27] make kerberoes enabled --- .../apache/kyuubi/KerberizedTestHelper.scala | 2 +- ...KyuubiOperationMultipleAuthTypeSuite.scala | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala index 31ed23112cf..ea032fdf820 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala @@ -48,7 +48,7 @@ trait KerberizedTestHelper extends KyuubiFunSuite { kdcConf.remove(MiniKdc.DEBUG) private var kdc: MiniKdc = _ - private var krb5ConfPath: String = _ + protected var krb5ConfPath: String = _ private val keytabFile = new File(baseDir, "kyuubi-test.keytab") protected val testKeytab: String = keytabFile.getAbsolutePath diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala index 16d9ed1a663..9006f734b35 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -19,6 +19,9 @@ package org.apache.kyuubi.operation import java.sql.{DriverManager, SQLException} +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.security.UserGroupInformation + import org.apache.kyuubi.{KerberizedTestHelper, WithKyuubiServer} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.UserDefineAuthenticationProviderImpl @@ -29,8 +32,27 @@ class KyuubiOperationMultipleAuthTypeSuite extends override protected def jdbcUrl: String = getJdbcUrl private def kerberosJdbcUrl: String = jdbcUrl + s"principal=${testPrincipal}" + private val currentUser = UserGroupInformation.getCurrentUser + + override def beforeAll(): Unit = { + super.beforeAll() + } + + override def afterAll(): Unit = { + System.clearProperty("java.security.krb5.conf") + UserGroupInformation.setLoginUser(currentUser) + UserGroupInformation.setConfiguration(new Configuration()) + assert(!UserGroupInformation.isSecurityEnabled) + } override protected lazy val conf: KyuubiConf = { + val config = new Configuration() + val authType = "hadoop.security.authentication" + config.set(authType, "KERBEROS") + System.setProperty("java.security.krb5.conf", krb5ConfPath) + UserGroupInformation.setConfiguration(config) + assert(UserGroupInformation.isSecurityEnabled) + KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "CUSTOM", "LDAP")) .set(KyuubiConf.SERVER_KEYTAB, testKeytab) .set(KyuubiConf.SERVER_PRINCIPAL, testPrincipal) From 2e2283baf18bafbc287006146cb41af0e3c7672e Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 00:35:26 +0800 Subject: [PATCH 19/27] after all --- .../kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala index 9006f734b35..9c109c66a18 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -43,6 +43,7 @@ class KyuubiOperationMultipleAuthTypeSuite extends UserGroupInformation.setLoginUser(currentUser) UserGroupInformation.setConfiguration(new Configuration()) assert(!UserGroupInformation.isSecurityEnabled) + super.afterAll() } override protected lazy val conf: KyuubiConf = { From d7cfaf4cbe14f98993ab751c85c4bdd6d5559434 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 00:52:57 +0800 Subject: [PATCH 20/27] only the first is valid --- .../KyuubiOperationMultipleAuthTypeSuite.scala | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala index 9c109c66a18..e718a20ae88 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala @@ -24,10 +24,10 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.kyuubi.{KerberizedTestHelper, WithKyuubiServer} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.service.authentication.UserDefineAuthenticationProviderImpl +import org.apache.kyuubi.service.authentication.{UserDefineAuthenticationProviderImpl, WithLdapServer} class KyuubiOperationMultipleAuthTypeSuite extends - WithKyuubiServer with KerberizedTestHelper with JDBCTestUtils { + WithKyuubiServer with KerberizedTestHelper with WithLdapServer with JDBCTestUtils { private val customPasswd: String = "password" override protected def jdbcUrl: String = getJdbcUrl @@ -59,6 +59,8 @@ class KyuubiOperationMultipleAuthTypeSuite extends .set(KyuubiConf.SERVER_PRINCIPAL, testPrincipal) .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) + .set(KyuubiConf.AUTHENTICATION_LDAP_URL, ldapUrl) + .set(KyuubiConf.AUTHENTICATION_LDAP_BASEDN, ldapBaseDn) } test("test with KERBEROS authentication") { @@ -85,6 +87,18 @@ class KyuubiOperationMultipleAuthTypeSuite extends } } + test("only the first plain auth type is valid") { + intercept[SQLException] { + val conn = DriverManager.getConnection(jdbcUrlWithConf, user, ldapUserPasswd) + try { + val statement = conn.createStatement() + statement.executeQuery("select engine_name()") + } finally { + conn.close() + } + } + } + test("test with invalid password") { intercept[SQLException] { val conn = DriverManager.getConnection(jdbcUrlWithConf, user, "invalidPassword") From d227aa74cb624e8208b64df7b439706acc49721b Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 01:02:56 +0800 Subject: [PATCH 21/27] fix ut --- .../test/scala/org/apache/kyuubi/WithSecuredDFSService.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala index 797232fc432..0bbbc3b092b 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSecuredDFSService.scala @@ -25,7 +25,7 @@ import org.apache.kyuubi.server.MiniDFSService trait WithSecuredDFSService extends KerberizedTestHelper { - private val miniDFSService = new MiniDFSService(newSecuredConf()) + private var miniDFSService: MiniDFSService = _ private def newSecuredConf(): Configuration = { val hdfsConf = new Configuration() @@ -45,12 +45,13 @@ trait WithSecuredDFSService extends KerberizedTestHelper { } override def beforeAll(): Unit = { + super.beforeAll() tryWithSecurityEnabled { UserGroupInformation.loginUserFromKeytab(testPrincipal, testKeytab) + miniDFSService = new MiniDFSService(newSecuredConf()) miniDFSService.initialize(new KyuubiConf(false)) miniDFSService.start() } - super.beforeAll() } override def afterAll(): Unit = { From 8d137db9970d3e9f7fa08e9373be0e4aba116577 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 01:26:59 +0800 Subject: [PATCH 22/27] make ldap password diff with custom --- .../apache/kyuubi/service/authentication/WithLdapServer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala index ea1fa138751..0bb38684e0b 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala @@ -25,7 +25,7 @@ trait WithLdapServer extends KyuubiFunSuite { protected var ldapServer: InMemoryDirectoryServer = _ protected val ldapBaseDn = "ou=users" protected val ldapUser = Utils.currentUser - protected val ldapUserPasswd = "password" + protected val ldapUserPasswd = "ldapPassword" protected def ldapUrl = s"ldap://localhost:${ldapServer.getListenPort}" From b1f83e55fc1236711e58d70d7bcf50807ab6c3af Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 09:51:06 +0800 Subject: [PATCH 23/27] add ut --- .../KyuubiAuthenticationFactorySuite.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index bdee687a430..6887a5eaae7 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -75,4 +75,16 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { val e = intercept[LoginException](factory.getTTransportFactory) assert(e.getMessage startsWith "Kerberos principal should have 3 parts") } + + test("AuthType is NOSASL if only NOSASL is specified") { + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NOSASL")) + var factory = new KyuubiAuthenticationFactory(conf) + factory.getTTransportFactory + assert(!Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) + + conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NOSASL", "NONE")) + factory = new KyuubiAuthenticationFactory(conf) + factory.getTTransportFactory + assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) + } } From 11f409cb10a4b3879136a9eee928b39f2fa315f8 Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 10:10:53 +0800 Subject: [PATCH 24/27] Update docs --- .../main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 2d0f261168a..178e2ab92c4 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -378,7 +378,11 @@ object KyuubiConf { "
  • NONE: no authentication check.
    • " + "
  • KERBEROS: Kerberos/GSSAPI authentication.
    • " + "
  • CUSTOM: User-defined authentication.
    • " + - "
  • LDAP: Lightweight Directory Access Protocol authentication.
    • ") + "
  • LDAP: Lightweight Directory Access Protocol authentication.
    • " + + " Note that: For KERBEROS, it is SASL/GSSAPI mechanism," + + " and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism." + + " If only NOSASL is specified, the authentication will be NOSASL." + + " For SASL authentication, KERBEROS and one PLAIN auth type are supported at the same time.") .version("1.0.0") .stringConf .toSequence() From ea7db79f8625cae6cbfee41edb3567cfa66f634f Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 10:28:25 +0800 Subject: [PATCH 25/27] complete --- docs/deployment/settings.md | 2 +- .../org/apache/kyuubi/config/KyuubiConf.scala | 3 ++- .../CustomAuthenticationProviderImplSuite.scala | 4 ++-- .../UserDefineAuthenticationProviderImpl.scala | 4 ++-- ...ubiOperationKerberosAndPlainAuthSuite.scala} | 17 +++++++++-------- 5 files changed, 16 insertions(+), 14 deletions(-) rename kyuubi-server/src/test/scala/org/apache/kyuubi/operation/{KyuubiOperationMultipleAuthTypeSuite.scala => KyuubiOperationKerberosAndPlainAuthSuite.scala} (90%) diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 204393631dc..d220a09ff45 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -122,7 +122,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co Key | Default | Meaning | Type | Since --- | --- | --- | --- | --- -kyuubi\.authentication|
    NONE
    |
    A comma separated list of client authentication types.
    • NOSASL: raw transport.
    • NONE: no authentication check.
    • KERBEROS: Kerberos/GSSAPI authentication.
    • CUSTOM: User-defined authentication.
    • LDAP: Lightweight Directory Access Protocol authentication.
    |
    seq
    |
    1.0.0
    +kyuubi\.authentication|
    NONE
    |
    A comma separated list of client authentication types.
    • NOSASL: raw transport.
    • NONE: no authentication check.
    • KERBEROS: Kerberos/GSSAPI authentication.
    • CUSTOM: User-defined authentication.
    • LDAP: Lightweight Directory Access Protocol authentication.
    Note that: For KERBEROS, it is SASL/GSSAPI mechanism, and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism. If only NOSASL is specified, the authentication will be NOSASL. For SASL authentication, KERBEROS and PLAIN auth type are supported at the same time, and only the first specified PLAIN auth type is valid.
    |
    seq
    |
    1.0.0
    kyuubi\.authentication
    \.custom\.class|
    <undefined>
    |
    User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider
    |
    string
    |
    1.3.0
    kyuubi\.authentication
    \.ldap\.base\.dn|
    <undefined>
    |
    LDAP base DN.
    |
    string
    |
    1.0.0
    kyuubi\.authentication
    \.ldap\.domain|
    <undefined>
    |
    LDAP domain.
    |
    string
    |
    1.0.0
    diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 178e2ab92c4..524448c5e8b 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -382,7 +382,8 @@ object KyuubiConf { " Note that: For KERBEROS, it is SASL/GSSAPI mechanism," + " and for NONE, CUSTOM and LDAP, they are all SASL/PLAIN mechanism." + " If only NOSASL is specified, the authentication will be NOSASL." + - " For SASL authentication, KERBEROS and one PLAIN auth type are supported at the same time.") + " For SASL authentication, KERBEROS and PLAIN auth type are supported at the same time," + + " and only the first specified PLAIN auth type is valid.") .version("1.0.0") .stringConf .toSequence() diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala index 64e599eb3c0..c5561d044fb 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/CustomAuthenticationProviderImplSuite.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.service.authentication import javax.security.sasl.AuthenticationException -import org.apache.kyuubi.{KyuubiFunSuite, Utils} +import org.apache.kyuubi.KyuubiFunSuite import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.AuthenticationProviderFactory.getAuthenticationProvider @@ -38,6 +38,6 @@ class CustomAuthenticationProviderImplSuite extends KyuubiFunSuite { val e2 = intercept[AuthenticationException](p1.authenticate("test", "test")) assert(e2.getMessage.contains("Username or password is not valid!")) - p1.authenticate(Utils.currentUser, "password") + p1.authenticate("user", "password") } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala index 181d009e326..7a84103cc46 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/UserDefineAuthenticationProviderImpl.scala @@ -19,13 +19,13 @@ package org.apache.kyuubi.service.authentication import javax.security.sasl.AuthenticationException -import org.apache.kyuubi.{Logging, Utils} +import org.apache.kyuubi.Logging class UserDefineAuthenticationProviderImpl() extends PasswdAuthenticationProvider with Logging { override def authenticate(user: String, password: String): Unit = { - if (user == Utils.currentUser && password == "password") { + if (user == "user" && password == "password") { info(s"Success log in of user: $user") } else { throw new AuthenticationException("Username or password is not valid!") diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationKerberosAndPlainAuthSuite.scala similarity index 90% rename from kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala rename to kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationKerberosAndPlainAuthSuite.scala index e718a20ae88..2e94d212992 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationMultipleAuthTypeSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationKerberosAndPlainAuthSuite.scala @@ -26,8 +26,9 @@ import org.apache.kyuubi.{KerberizedTestHelper, WithKyuubiServer} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.{UserDefineAuthenticationProviderImpl, WithLdapServer} -class KyuubiOperationMultipleAuthTypeSuite extends +class KyuubiOperationKerberosAndPlainAuthSuite extends WithKyuubiServer with KerberizedTestHelper with WithLdapServer with JDBCTestUtils { + private val customUser: String = "user" private val customPasswd: String = "password" override protected def jdbcUrl: String = getJdbcUrl @@ -54,13 +55,13 @@ class KyuubiOperationMultipleAuthTypeSuite extends UserGroupInformation.setConfiguration(config) assert(UserGroupInformation.isSecurityEnabled) - KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "CUSTOM", "LDAP")) + KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS", "LDAP", "CUSTOM")) .set(KyuubiConf.SERVER_KEYTAB, testKeytab) .set(KyuubiConf.SERVER_PRINCIPAL, testPrincipal) - .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, - classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) .set(KyuubiConf.AUTHENTICATION_LDAP_URL, ldapUrl) .set(KyuubiConf.AUTHENTICATION_LDAP_BASEDN, ldapBaseDn) + .set(KyuubiConf.AUTHENTICATION_CUSTOM_CLASS, + classOf[UserDefineAuthenticationProviderImpl].getCanonicalName) } test("test with KERBEROS authentication") { @@ -75,8 +76,8 @@ class KyuubiOperationMultipleAuthTypeSuite extends } } - test("test with CUSTOM authentication") { - val conn = DriverManager.getConnection(jdbcUrlWithConf, user, customPasswd) + test("test with LDAP authentication") { + val conn = DriverManager.getConnection(jdbcUrlWithConf, ldapUser, ldapUserPasswd) try { val statement = conn.createStatement() val resultSet = statement.executeQuery("select engine_name()") @@ -87,9 +88,9 @@ class KyuubiOperationMultipleAuthTypeSuite extends } } - test("only the first plain auth type is valid") { + test("only the first specified plain auth type is valid") { intercept[SQLException] { - val conn = DriverManager.getConnection(jdbcUrlWithConf, user, ldapUserPasswd) + val conn = DriverManager.getConnection(jdbcUrlWithConf, customUser, customPasswd) try { val statement = conn.createStatement() statement.executeQuery("select engine_name()") From 850d6b5d785c06407af57707df843297784a8a3e Mon Sep 17 00:00:00 2001 From: fwang12 Date: Sun, 24 Oct 2021 10:43:32 +0800 Subject: [PATCH 26/27] fix ut --- .../authentication/KyuubiAuthenticationFactorySuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index 6887a5eaae7..2298c96e32d 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -20,6 +20,8 @@ package org.apache.kyuubi.service.authentication import java.security.Security import javax.security.auth.login.LoginException +import org.apache.thrift.transport.TSaslServerTransport + import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider @@ -79,12 +81,10 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { test("AuthType is NOSASL if only NOSASL is specified") { val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NOSASL")) var factory = new KyuubiAuthenticationFactory(conf) - factory.getTTransportFactory - assert(!Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) + !factory.getTTransportFactory.isInstanceOf[TSaslServerTransport.Factory] conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NOSASL", "NONE")) factory = new KyuubiAuthenticationFactory(conf) - factory.getTTransportFactory - assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) + factory.getTTransportFactory.isInstanceOf[TSaslServerTransport.Factory] } } From 71053aefca3af1c668d656c2119fa732b732b28f Mon Sep 17 00:00:00 2001 From: fwang12 Date: Mon, 25 Oct 2021 11:13:40 +0800 Subject: [PATCH 27/27] adress nit --- .../test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala index 2cac5b9dd70..0abcbaf3a5d 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestUtils.scala @@ -63,7 +63,7 @@ trait JDBCTestUtils extends KyuubiFunSuite { } } - protected def jdbcUrlWithConf(): String = jdbcUrlWithConf(jdbcUrl) + protected def jdbcUrlWithConf: String = jdbcUrlWithConf(jdbcUrl) protected def jdbcUrlWithConf(jdbcUrl: String): String = { val sessionConfStr = sessionConfigs.map(kv => kv._1 + "=" + kv._2).mkString(";")