[SPARK-31971][WEBUI] Add pagination support for all jobs timeline

core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala

@@ -162,7 +162,11 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
   private def makeTimeline(
       jobs: Seq[v1.JobData],
       executors: Seq[v1.ExecutorSummary],
-      startTime: Long): Seq[Node] = {
+      startTime: Long,
+      page: Int,
+      pageSize: Int,
+      totalPages: Int,
+      totalJobs: Int): Seq[Node] = {
 
     val jobEventJsonAsStrSeq = makeJobEvent(jobs)
     val executorEventJsonAsStrSeq = makeExecutorEvent(executors)
@@ -184,20 +188,52 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
     val eventArrayAsStr =
       (jobEventJsonAsStrSeq ++ executorEventJsonAsStrSeq).mkString("[", ",", "]")
 
-    <span class="expand-application-timeline">
-      <span class="expand-application-timeline-arrow arrow-closed"></span>
-      <a data-toggle="tooltip" title={ToolTips.JOB_TIMELINE} data-placement="top">
-        Event Timeline
-      </a>
-    </span> ++
-    <div id="application-timeline" class="collapsed">
-      <div class="control-panel">
-        <div id="application-timeline-zoom-lock">
-          <input type="checkbox"></input>
-          <span>Enable zooming</span>
+    if (totalPages > 0) {
+      <span class="expand-application-timeline">
+        <span class="expand-application-timeline-arrow arrow-closed"></span>
+        <a data-toggle="tooltip" title={ToolTips.JOB_TIMELINE} data-placement="top">
+          Event Timeline
+        </a>
+      </span> ++
+        <div id="application-timeline" class="collapsed">
+          <div class="control-panel">
+            <div id="application-timeline-zoom-lock">
+              <input type="checkbox"></input>
+              <span>Enable zooming</span>
+            </div>
+            <div>
+              <form id={s"form-event-timeline-page"}
+                    method="get"
+                    action=""
+                    class="form-inline justify-content-end"
+                    style="width: 50%; margin-left: auto; margin-bottom: 0px;">
+                <label>Jobs:
+                  {totalJobs}
+                  .
+                  {totalPages}
+                  Pages. Jump to</label>
+                <input type="text"
+                       name="jobs.eventTimelinePageNumber"
+                       id={s"form-event-timeline-page-no"}
+                       value={page.toString}
+                       class="col-1 form-control"/>
+                <label>. Show</label>
+                <input type="text"
+                       id={s"form-event-timeline-page-size"}
+                       name="jobs.eventTimelinePageSize"
+                       value={pageSize.toString}
+                       class="col-1 form-control"/>
+                <label>items in a page.</label>
+                <button type="submit" id="form-event-timeline-page-button" class="btn btn-spark">
+                  Go
+                </button>
+              </form>
+            </div>
+          </div>
         </div>
-      </div>
-    </div> ++
+    } else {
+      Seq.empty
+    } ++
     <script type="text/javascript">
       {Unparsed(s"drawApplicationTimeline(${groupJsonArrayAsStr}," +
       s"${eventArrayAsStr}, ${startTime}, ${UIUtils.getTimeZoneOffset()});")}
@@ -258,6 +294,25 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       }
     }
 
+    val eventTimelineParameterJobsPage = request.getParameter("jobs.eventTimelinePageNumber")
+    val eventTimelineParameterJobsPageSize = request.getParameter("jobs.eventTimelinePageSize")
+    var eventTimelineJobsPage = Option(eventTimelineParameterJobsPage).map(_.toInt).getOrElse(1)
+    var eventTimelineJobsPageSize =
+      Option(eventTimelineParameterJobsPageSize).map(_.toInt).getOrElse(100)
+
+    val totalJobs = completedJobs.size + failedJobs.size + activeJobs.size
+    if (eventTimelineJobsPageSize < 1 || eventTimelineJobsPageSize > totalJobs) {
+      eventTimelineJobsPageSize = totalJobs
+    }
+    val eventTimelineTotalPages = if (eventTimelineJobsPageSize > 0) {
+      (totalJobs + eventTimelineJobsPageSize - 1) / eventTimelineJobsPageSize
+    } else {
+      0
+    }
+    if (eventTimelineJobsPage < 1 || eventTimelineJobsPage > eventTimelineTotalPages) {
+      eventTimelineJobsPage = 1
+    }
+
     val activeJobsTable =
       jobsTable(request, "active", "activeJob", activeJobs, killEnabled = parent.killEnabled)
     val completedJobsTable =
@@ -329,9 +384,16 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
         </ul>
       </div>
 
+    val from = (eventTimelineJobsPage - 1) * eventTimelineJobsPageSize
+    val to = from + eventTimelineJobsPageSize
     var content = summary
-    content ++= makeTimeline(activeJobs ++ completedJobs ++ failedJobs,
-      store.executorList(false), startTime)
+    content ++= makeTimeline(
+      (activeJobs ++ completedJobs ++ failedJobs).sortBy(_.submissionTime).slice(from, to),
+      store.executorList(false),
+      startTime, eventTimelineJobsPage,
+      eventTimelineJobsPageSize,
+      eventTimelineTotalPages,
+      totalJobs)
 
     if (shouldShowActiveJobs) {
       content ++=

core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala

@@ -17,7 +17,7 @@
 
 package org.apache.spark.ui
 
-import org.openqa.selenium.WebDriver
+import org.openqa.selenium.{JavascriptExecutor, WebDriver}
 import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
 
 import org.apache.spark.tags.ChromeUITest
@@ -28,7 +28,7 @@ import org.apache.spark.tags.ChromeUITest
 @ChromeUITest
 class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") {
 
-  override var webDriver: WebDriver = _
+  override var webDriver: WebDriver with JavascriptExecutor = _
 
   override def beforeAll(): Unit = {
     super.beforeAll()

core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala

@@ -17,7 +17,13 @@
 
 package org.apache.spark.ui
 
-import org.openqa.selenium.{By, WebDriver}
+import java.util.{List => JList}
+import java.util.regex._
+
+import scala.collection.JavaConverters._
+import scala.util.Try
+
+import org.openqa.selenium.{By, JavascriptExecutor, WebDriver}
 import org.scalatest._
 import org.scalatest.concurrent.Eventually._
 import org.scalatest.time.SpanSugar._
@@ -35,7 +41,7 @@ import org.apache.spark.util.CallSite
 abstract class RealBrowserUISeleniumSuite(val driverProp: String)
   extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
 
-  implicit var webDriver: WebDriver
+  implicit var webDriver: WebDriver with JavascriptExecutor
   private val driverPropPrefix = "spark.test."
 
   override def beforeAll(): Unit = {
@@ -128,6 +134,108 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String)
     }
   }
 
+  test("Pagination for all jobs timeline") {
+    val totalNumOfJobs = 301
+    withSpark(newSparkContext()) { sc =>
+      (0 until totalNumOfJobs).foreach { index =>
+        // Just ignore exception.
+        Try {
+          sc.parallelize(1 to 10).foreach { _ =>
+            if (index == 0) {
+              // Mark the first job fail.
+              throw new RuntimeException()
+            }
+          }
+        }
+      }
+
+      eventually(timeout(10 seconds), interval(50 milliseconds)) {
+        goToUi(sc, "/jobs")
+        webDriver.findElement(By.cssSelector("span.expand-application-timeline")).click()
+
+        // The number of initially displayed jobs.
+        val displayedJobs1 = getJobContents()
+        displayedJobs1.size should be (100)
+
+        // Jobs are sorted by submission time (equivalent to Job ID in this case)
+        // regardless of whether jobs are completed or not.
+        val pattern = Pattern.compile("""^.*\(Job (\d+)\)$""")
+        displayedJobs1.sliding(2).foreach { case Seq(content1, content2) =>
+          val matcher1 = pattern.matcher(content1)
+          matcher1.find()
+          val jobId1 = matcher1.group(1).toInt
+
+          val matcher2 = pattern.matcher(content2)
+          matcher2.find()
+          val jobId2 = matcher2.group(1).toInt
+
+          jobId1 should be < jobId2
+        }
+
+        goToPage(pageNo = 3, pageSize = 30)
+        val displayedJobs2 = getJobContents()
+        displayedJobs2.size should be (30)
+
+        // If designated pageNo exceeds the num of total pages or < 1,
+        // the pageNo is set to 1 but pageSize is kept.
+        goToPage(pageNo = 100, pageSize = 50)
+        val displayedJobs3 = getJobContents()
+        displayedJobs3.length should be (50)
+        displayedJobs3.indices.foreach { index =>
+          displayedJobs3(index) should be (displayedJobs1.slice(0, 50)(index))
+        }
+        goToPage(pageNo = -1, pageSize = 10)
+        val displayedJobs4 = getJobContents()
+        displayedJobs4.indices.foreach { index =>
+          displayedJobs4(index) should be (displayedJobs1.slice(0, 10)(index))
+        }
+
+        // If designated pageSize exceeds the num of jobs or < 1,
+        // the pageSize is set to the num of jobs.
+        goToPage(pageNo = 2, pageSize = 10000)
+        val displayedJobs5 = getJobContents()
+        displayedJobs5.size should be (totalNumOfJobs)
+        goToPage(pageNo = 5, pageSize = 0)
+        val displayedJobs6 = getJobContents()
+        displayedJobs6.size should be (totalNumOfJobs)
+
+        // The num of jobs in the last page at a pageSize can be < pageSize
+        goToPage(pageNo = 4, pageSize = 100)
+        val displayedJobs7 = getJobContents()
+        displayedJobs7.size should be (totalNumOfJobs % 100)
+      }
+    }
+
+    // WebElement#sendKeys cannot work on macOS with ChromeDriver so executeScript is used.
+    def setPageNo(pageNo: Int): Unit = {
+      webDriver.executeScript(s"$$('#form-event-timeline-page-no').attr('value', '$pageNo');")
+    }
+
+    def setPageSize(pageSize: Int): Unit = {
+      webDriver.executeScript(s"$$('#form-event-timeline-page-size').attr('value', '$pageSize');")
+    }
+
+    // If text fields are filled by executeScript, Go button should be pushed by the method.
+    def pushGoButton(): Unit = {
+      webDriver.executeScript("$('#form-event-timeline-page-button').click();")
+    }
+
+    def getJobContents(): Seq[String] = {
+      webDriver.executeScript(
+        """
+          return $('.vis-item.job').map(function(_, e) {
+            return e.textContent;
+          });
+        """).asInstanceOf[JList[String]].asScala
+    }
+
+    def goToPage(pageNo: Int, pageSize: Int): Unit = {
+      setPageNo(pageNo)
+      setPageSize(pageSize)
+      pushGoButton()
+    }
+  }
+
   /**
    * Create a test SparkContext with the SparkUI enabled.
    * It is safe to `get` the SparkUI directly from the SparkContext returned here.