From b5730a1ed154893b926ff9656a287fb847d3a407 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sun, 21 Dec 2014 23:55:38 +0800 Subject: [PATCH 1/2] Fix incorrect event log path. --- .../main/scala/org/apache/spark/deploy/master/Master.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index ed5eab9f473c..21da9609e22c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -650,7 +650,11 @@ private[spark] class Master( def createApplication(desc: ApplicationDescription, driver: ActorRef): ApplicationInfo = { val now = System.currentTimeMillis() val date = new Date(now) - new ApplicationInfo(now, newApplicationId(date), desc, date, driver, defaultCores) + val appId = newApplicationId(date) + val logPath = EventLoggingListener.getLogPath(desc.eventLogFile.get, appId) + val newAppDesc = new ApplicationDescription(desc.name, desc.maxCores, desc.memoryPerSlave, + desc.command, desc.appUiUrl, Some(logPath)) + new ApplicationInfo(now, appId, newAppDesc, date, driver, defaultCores) } def registerApplication(app: ApplicationInfo): Unit = { From 5e0ea35d2d1c7a92343578b656d98b9c98d7a6fb Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 23 Dec 2014 15:16:16 +0800 Subject: [PATCH 2/2] Revision for comment. --- .../spark/deploy/ApplicationDescription.scala | 2 +- .../apache/spark/deploy/master/Master.scala | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala b/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala index b10b7b8d3252..65a1a8fd7e92 100644 --- a/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala +++ b/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala @@ -23,7 +23,7 @@ private[spark] class ApplicationDescription( val memoryPerSlave: Int, val command: Command, var appUiUrl: String, - val eventLogFile: Option[String] = None) + val eventLogDir: Option[String] = None) extends Serializable { val user = System.getProperty("user.name", "") diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index 21da9609e22c..f21c3e290331 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -650,11 +650,7 @@ private[spark] class Master( def createApplication(desc: ApplicationDescription, driver: ActorRef): ApplicationInfo = { val now = System.currentTimeMillis() val date = new Date(now) - val appId = newApplicationId(date) - val logPath = EventLoggingListener.getLogPath(desc.eventLogFile.get, appId) - val newAppDesc = new ApplicationDescription(desc.name, desc.maxCores, desc.memoryPerSlave, - desc.command, desc.appUiUrl, Some(logPath)) - new ApplicationInfo(now, appId, newAppDesc, date, driver, defaultCores) + new ApplicationInfo(now, newApplicationId(date), desc, date, driver, defaultCores) } def registerApplication(app: ApplicationInfo): Unit = { @@ -718,14 +714,26 @@ private[spark] class Master( def rebuildSparkUI(app: ApplicationInfo): Boolean = { val appName = app.desc.name val notFoundBasePath = HistoryServer.UI_PATH_PREFIX + "/not-found" - val eventLogFile = app.desc.eventLogFile.getOrElse { - // Event logging is not enabled for this application - app.desc.appUiUrl = notFoundBasePath + val eventLogFile = app.desc.eventLogDir + .map { dir => EventLoggingListener.getLogPath(dir, app.id) } + .getOrElse { + // Event logging is not enabled for this application + app.desc.appUiUrl = notFoundBasePath + return false + } + val fs = Utils.getHadoopFileSystem(eventLogFile, hadoopConf) + + if (fs.exists(new Path(eventLogFile + EventLoggingListener.IN_PROGRESS))) { + // Event logging is enabled for this application, but the application is still in progress + val title = s"Application history not found (${app.id})" + var msg = s"Application $appName is still in progress." + logWarning(msg) + msg = URLEncoder.encode(msg, "UTF-8") + app.desc.appUiUrl = notFoundBasePath + s"?msg=$msg&title=$title" return false } try { - val fs = Utils.getHadoopFileSystem(eventLogFile, hadoopConf) val (logInput, sparkVersion) = EventLoggingListener.openEventLog(new Path(eventLogFile), fs) val replayBus = new ReplayListenerBus() val ui = SparkUI.createHistoryUI(new SparkConf, replayBus, new SecurityManager(conf),