From 4d7c43373d43126b488540d7659274277665f51c Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Mon, 25 Jan 2016 16:43:40 -0800 Subject: [PATCH 1/4] simulated infinite scrolling using current pagination code --- .../org/apache/spark/ui/static/webui.css | 4 ++ .../spark/deploy/worker/ui/LogPage.scala | 59 +++++++++++-------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css index 48f86d1536c9..42f725d2d6bd 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.css +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css @@ -235,3 +235,7 @@ a.expandbutton { color: #333; text-decoration: none; } + +.logMoreBtn { + width: 100% +} \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala index 49803a27a5b0..b9be6f7e2aaa 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala @@ -21,7 +21,7 @@ import java.io.File import java.net.URI import javax.servlet.http.HttpServletRequest -import scala.xml.Node +import scala.xml.{Node, Unparsed} import org.apache.spark.Logging import org.apache.spark.ui.{UIUtils, WebUIPage} @@ -77,49 +77,62 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) val linkToMaster =

Back to Master

- val range = Bytes {startByte.toString} - {endByte.toString} of {logLength} + val range = + + Showing {byteLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength} + - val backButton = + val moreButton = if (startByte > 0) { - } else { - } - val nextButton = + val newButton = if (endByte < logLength) { - - } else { - + Seq.empty + } + + val scrollPercent = + if (!offset.isEmpty) { + defaultBytes.toDouble / byteLength + } else { + 1.0 } + val script = + + val content = {linkToMaster} -
-
{backButton}
-
{range}
-
{nextButton}
-
-
-
+ {range} +
+
{moreButton}
{logText}
+
{newButton}
+ {script} UIUtils.basicSparkPage(content, logType + " log page for " + pageName) From f37fa32edaea1075c9f8f5497e896285917548c6 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Fri, 29 Jan 2016 20:29:23 -0800 Subject: [PATCH 2/4] updated log page to use javascript for infinite scrolling --- .../org/apache/spark/ui/static/log-view.js | 125 ++++++++++++++++++ .../org/apache/spark/ui/static/webui.css | 8 +- .../spark/deploy/worker/ui/LogPage.scala | 65 +++------ .../scala/org/apache/spark/ui/UIUtils.scala | 1 + 4 files changed, 155 insertions(+), 44 deletions(-) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/log-view.js diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js new file mode 100644 index 000000000000..342bc808e0ea --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var baseParams + +var curLogLength +var startByte +var endByte +var totalLogLength + +var byteLength +var btnHeight = 30 + +function setLogScroll(oldHeight) { + $(".log-content").scrollTop($(".log-content")[0].scrollHeight - oldHeight); +} + +function tailLog() { + $(".log-content").scrollTop($(".log-content")[0].scrollHeight); +} + +function setLogData() { + $('#log-data').html("Showing " + curLogLength + " Bytes: " + startByte + + " - " + endByte + " of " + totalLogLength); +} + +function disableMoreButton() { + $(".log-more-btn").attr("disabled", "disabled");; + $(".log-more-btn").html("Top of Log"); +} + +function noNewAlert() { + $(".no-new-alert").css("display", "block"); + window.setTimeout(function () {$(".no-new-alert").css("display", "none");}, 4000); +} + +function loadMore() { + var offset = Math.max(startByte - byteLength, 0); + var newLogLength = Math.min(curLogLength + byteLength, totalLogLength); + + $.ajax({ + type: "GET", + url: "/log" + baseParams + "&offset=" + offset + "&byteLength=" + byteLength, + success: function (data) { + var oldHeight = $(".log-content")[0].scrollHeight; + var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); + var retStartByte = dataInfo[0]; + var retLogLength = dataInfo[2]; + + var cleanData = data.substring(data.indexOf('\n') + 1).trim(); + if (retStartByte == 0) { + cleanData = cleanData.substring(0, startByte); + disableMoreButton(); + } + $("pre", ".log-content").prepend(cleanData); + + curLogLength = curLogLength + (startByte - retStartByte); + startByte = retStartByte; + totalLogLength = retLogLength; + setLogScroll(oldHeight); + setLogData(); + } + }); +} + +function loadNew() { + $.ajax({ + type: "GET", + url: "/log" + baseParams, + success: function (data) { + var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); + var newDataLen = dataInfo[2] - totalLogLength; + if (newDataLen != 0) { + $.ajax({ + type: "GET", + url: "/log" + baseParams + "&byteLength=" + newDataLen, + success: function (data) { + var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); + var retStartByte = dataInfo[0]; + var retEndByte = dataInfo[1]; + var retLogLength = dataInfo[2]; + + var cleanData = data.substring(data.indexOf('\n') + 1).trim(); + $("pre", ".log-content").append(cleanData); + + curLogLength = curLogLength + (retEndByte - retStartByte); + endByte = retEndByte; + totalLogLength = retLogLength; + tailLog(); + setLogData(); + } + }); + } else { + noNewAlert(); + } + } + }); +} + +function initLogPage(params, logLen, start, end, totLogLen, defaultLen) { + baseParams = params; + curLogLength = logLen; + startByte = start; + endByte = end; + totalLogLength = totLogLen; + byteLength = defaultLen; + tailLog(); + if (startByte == 0) { + disableMoreButton(); + } +} \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.css b/core/src/main/resources/org/apache/spark/ui/static/webui.css index 42f725d2d6bd..48d964e87c8a 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.css +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.css @@ -236,6 +236,12 @@ a.expandbutton { text-decoration: none; } -.logMoreBtn { +.log-more-btn, .log-new-btn { width: 100% +} + +.no-new-alert { + text-align: center; + margin: 0; + padding: 4px 0; } \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala index b9be6f7e2aaa..b1eab60f6021 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala @@ -32,10 +32,9 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with private val worker = parent.worker private val workDir = parent.workDir private val supportedLogTypes = Set("stderr", "stdout") + private val defaultBytes = 100 * 1024 def renderLog(request: HttpServletRequest): String = { - val defaultBytes = 100 * 1024 - val appId = Option(request.getParameter("appId")) val executorId = Option(request.getParameter("executorId")) val driverId = Option(request.getParameter("driverId")) @@ -45,9 +44,9 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with val logDir = (appId, executorId, driverId) match { case (Some(a), Some(e), None) => - s"${workDir.getPath}/$appId/$executorId/" + s"${workDir.getPath}/$a/$e/" case (None, None, Some(d)) => - s"${workDir.getPath}/$driverId/" + s"${workDir.getPath}/$d/" case _ => throw new Exception("Request must specify either application or driver identifiers") } @@ -58,7 +57,6 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with } def render(request: HttpServletRequest): Seq[Node] = { - val defaultBytes = 100 * 1024 val appId = Option(request.getParameter("appId")) val executorId = Option(request.getParameter("executorId")) val driverId = Option(request.getParameter("driverId")) @@ -77,62 +75,43 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) val linkToMaster =

Back to Master

+ val curLogLength = endByte - startByte; val range = - - Showing {byteLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength} + + Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength} val moreButton = - if (startByte > 0) { - - - - } else { - - } + val newButton = - if (endByte < logLength) { - - - - } else { - Seq.empty - } + - val scrollPercent = - if (!offset.isEmpty) { - defaultBytes.toDouble / byteLength - } else { - 1.0 - } + val alert = + - val script = - + val logParams = "?%s&logType=%s".format(params, logType) + val jsOnload = "window.onload = " + + s"initLogPage('$logParams', $curLogLength, $startByte, $endByte, $logLength, $byteLength);" val content = {linkToMaster} {range} -
+
{moreButton}
{logText}
+ {alert}
{newButton}
- {script} + UIUtils.basicSparkPage(content, logType + " log page for " + pageName) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 1949c4b3cbf4..b43917d5fa75 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -168,6 +168,7 @@ private[spark] object UIUtils extends Logging { + } def vizHeaderNodes: Seq[Node] = { From 0f49738de032c78873063f29cf60c060ab3b04ef Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Thu, 7 Apr 2016 15:03:34 -0700 Subject: [PATCH 3/4] Addressed js comments --- .../org/apache/spark/ui/static/log-view.js | 46 ++++++++++--------- .../org/apache/spark/ui/JettyUtils.scala | 4 +- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js index 342bc808e0ea..f023cbfd4848 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js +++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js @@ -15,22 +15,23 @@ * limitations under the License. */ -var baseParams +var baseParams; -var curLogLength -var startByte -var endByte -var totalLogLength +var curLogLength; +var startByte; +var endByte; +var totalLogLength; -var byteLength -var btnHeight = 30 +var byteLength; function setLogScroll(oldHeight) { - $(".log-content").scrollTop($(".log-content")[0].scrollHeight - oldHeight); + var logContent = $(".log-content"); + logContent.scrollTop(logContent[0].scrollHeight - oldHeight); } function tailLog() { - $(".log-content").scrollTop($(".log-content")[0].scrollHeight); + var logContent = $(".log-content"); + logContent.scrollTop(logContent[0].scrollHeight); } function setLogData() { @@ -39,31 +40,33 @@ function setLogData() { } function disableMoreButton() { - $(".log-more-btn").attr("disabled", "disabled");; - $(".log-more-btn").html("Top of Log"); + var moreBtn = $(".log-more-btn"); + moreBtn.attr("disabled", "disabled"); + moreBtn.html("Top of Log"); } function noNewAlert() { - $(".no-new-alert").css("display", "block"); - window.setTimeout(function () {$(".no-new-alert").css("display", "none");}, 4000); + var alert = $(".no-new-alert") + alert.css("display", "block"); + window.setTimeout(function () {alert.css("display", "none");}, 4000); } function loadMore() { var offset = Math.max(startByte - byteLength, 0); - var newLogLength = Math.min(curLogLength + byteLength, totalLogLength); + var moreByteLength = Math.min(byteLength, startByte); $.ajax({ type: "GET", - url: "/log" + baseParams + "&offset=" + offset + "&byteLength=" + byteLength, + url: "/log" + baseParams + "&offset=" + offset + "&byteLength=" + moreByteLength, success: function (data) { var oldHeight = $(".log-content")[0].scrollHeight; - var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); + var newlineIndex = data.indexOf('\n') + var dataInfo = data.substring(0, newlineIndex).match(/\d+/g); var retStartByte = dataInfo[0]; var retLogLength = dataInfo[2]; - var cleanData = data.substring(data.indexOf('\n') + 1).trim(); + var cleanData = data.substring(newlineIndex + 1); if (retStartByte == 0) { - cleanData = cleanData.substring(0, startByte); disableMoreButton(); } $("pre", ".log-content").prepend(cleanData); @@ -80,7 +83,7 @@ function loadMore() { function loadNew() { $.ajax({ type: "GET", - url: "/log" + baseParams, + url: "/log" + baseParams + "&byteLength=0", success: function (data) { var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); var newDataLen = dataInfo[2] - totalLogLength; @@ -89,12 +92,13 @@ function loadNew() { type: "GET", url: "/log" + baseParams + "&byteLength=" + newDataLen, success: function (data) { - var dataInfo = data.substring(0, data.indexOf('\n')).match(/\d+/g); + var newlineIndex = data.indexOf('\n') + var dataInfo = data.substring(0, newlineIndex).match(/\d+/g); var retStartByte = dataInfo[0]; var retEndByte = dataInfo[1]; var retLogLength = dataInfo[2]; - var cleanData = data.substring(data.indexOf('\n') + 1).trim(); + var cleanData = data.substring(newlineIndex + 1); $("pre", ".log-content").append(cleanData); curLogLength = curLogLength + (retEndByte - retStartByte); diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index c3c59f857dc4..e22583e54b10 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -83,9 +83,7 @@ private[spark] object JettyUtils extends Logging { val result = servletParams.responder(request) response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate") response.setHeader("X-Frame-Options", xFrameOptionsValue) - // scalastyle:off println - response.getWriter.println(servletParams.extractFn(result)) - // scalastyle:on println + response.getWriter.print(servletParams.extractFn(result)) } else { response.setStatus(HttpServletResponse.SC_UNAUTHORIZED) response.setHeader("Cache-Control", "no-cache, no-store, must-revalidate") From 293f1a4ebdd355152127e8089c2520ce0ff1bf15 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Mon, 18 Apr 2016 08:04:42 -0700 Subject: [PATCH 4/4] fixed some semicolons --- .../main/resources/org/apache/spark/ui/static/log-view.js | 6 +++--- .../scala/org/apache/spark/deploy/worker/ui/LogPage.scala | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js index f023cbfd4848..1782b4f209c0 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js +++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js @@ -46,7 +46,7 @@ function disableMoreButton() { } function noNewAlert() { - var alert = $(".no-new-alert") + var alert = $(".no-new-alert"); alert.css("display", "block"); window.setTimeout(function () {alert.css("display", "none");}, 4000); } @@ -60,7 +60,7 @@ function loadMore() { url: "/log" + baseParams + "&offset=" + offset + "&byteLength=" + moreByteLength, success: function (data) { var oldHeight = $(".log-content")[0].scrollHeight; - var newlineIndex = data.indexOf('\n') + var newlineIndex = data.indexOf('\n'); var dataInfo = data.substring(0, newlineIndex).match(/\d+/g); var retStartByte = dataInfo[0]; var retLogLength = dataInfo[2]; @@ -92,7 +92,7 @@ function loadNew() { type: "GET", url: "/log" + baseParams + "&byteLength=" + newDataLen, success: function (data) { - var newlineIndex = data.indexOf('\n') + var newlineIndex = data.indexOf('\n'); var dataInfo = data.substring(0, newlineIndex).match(/\d+/g); var retStartByte = dataInfo[0]; var retEndByte = dataInfo[1]; diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala index 97c2c3de6cf6..3473c41b935f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala @@ -74,7 +74,7 @@ private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage") with val (logText, startByte, endByte, logLength) = getLog(logDir, logType, offset, byteLength) val linkToMaster =

Back to Master

- val curLogLength = endByte - startByte; + val curLogLength = endByte - startByte val range = Showing {curLogLength} Bytes: {startByte.toString} - {endByte.toString} of {logLength}