diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index 7e21fa681aa1..2b00a4a6b3e5 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -269,7 +269,7 @@ object HistoryServer extends Logging { Utils.initDaemon(log) new HistoryServerArguments(conf, argStrings) initSecurity() - val securityManager = new SecurityManager(conf) + val securityManager = createSecurityManager(conf) val providerName = conf.getOption("spark.history.provider") .getOrElse(classOf[FsHistoryProvider].getName()) @@ -289,6 +289,21 @@ object HistoryServer extends Logging { while(true) { Thread.sleep(Int.MaxValue) } } + /** + * Create a security manager. + * This turns off security in the SecurityManager, so that the the History Server can start + * in a Spark cluster where security is enabled. + * @param config configuration for the SecurityManager constructor + * @return the security manager for use in constructing the History Server. + */ + private[history] def createSecurityManager(config: SparkConf): SecurityManager = { + if (config.getBoolean(SecurityManager.SPARK_AUTH_CONF, false)) { + logDebug(s"Clearing ${SecurityManager.SPARK_AUTH_CONF}") + config.set(SecurityManager.SPARK_AUTH_CONF, "false") + } + new SecurityManager(config) + } + def initSecurity() { // If we are accessing HDFS and it has security enabled (Kerberos), we have to login // from a keytab file so that we can access HDFS beyond the kerberos ticket expiration. diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 715811a46f42..d3b79dd3e36e 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -75,7 +75,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers .set("spark.testing", "true") provider = new FsHistoryProvider(conf) provider.checkForLogs() - val securityManager = new SecurityManager(conf) + val securityManager = HistoryServer.createSecurityManager(conf) server = new HistoryServer(conf, provider, securityManager, 18080) server.initialize() @@ -288,7 +288,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers provider = new FsHistoryProvider(conf) provider.checkForLogs() - val securityManager = new SecurityManager(conf) + val securityManager = HistoryServer.createSecurityManager(conf) server = new HistoryServer(conf, provider, securityManager, 18080) server.initialize() @@ -349,6 +349,17 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers } + /** + * Verify that the security manager needed for the history server can be instantiated + * when `spark.authenticate` is `true`, rather than raise an `IllegalArgumentException`. + */ + test("security manager starts with spark.authenticate set") { + val conf = new SparkConf() + .set("spark.testing", "true") + .set(SecurityManager.SPARK_AUTH_CONF, "true") + HistoryServer.createSecurityManager(conf) + } + test("incomplete apps get refreshed") { implicit val webDriver: WebDriver = new HtmlUnitDriver @@ -368,7 +379,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers .set("spark.history.cache.window", "250ms") .remove("spark.testing") val provider = new FsHistoryProvider(myConf) - val securityManager = new SecurityManager(myConf) + val securityManager = HistoryServer.createSecurityManager(myConf) sc = new SparkContext("local", "test", myConf) val logDirUri = logDir.toURI