From 1e4a18616a41dfbb4d6e58134facd5e593846af5 Mon Sep 17 00:00:00 2001 From: guoxiaolong Date: Wed, 10 Jan 2018 16:16:40 +0800 Subject: [PATCH 1/4] [SPARK-23024][WEB-UI]Spark ui about the contents of the form need to have hidden and show features, when the table records very much. --- .../deploy/master/ui/ApplicationPage.scala | 19 +++++-- .../spark/deploy/master/ui/MasterPage.scala | 47 ++++++++++++---- .../spark/deploy/worker/ui/WorkerPage.scala | 40 +++++++++++--- .../apache/spark/ui/env/EnvironmentPage.scala | 36 +++++++++++-- .../apache/spark/ui/jobs/AllJobsPage.scala | 30 ++++++++--- .../apache/spark/ui/jobs/AllStagesPage.scala | 50 +++++++++++++---- .../org/apache/spark/ui/jobs/JobPage.scala | 53 +++++++++++++++---- .../org/apache/spark/ui/jobs/PoolPage.scala | 9 +++- .../org/apache/spark/ui/jobs/StagePage.scala | 9 +++- .../apache/spark/ui/storage/StoragePage.scala | 9 +++- 10 files changed, 246 insertions(+), 56 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala index 68e57b7564ad..4d15b690a07d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala @@ -100,12 +100,23 @@ private[ui] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app")
-

Executor Summary ({allExecutors.length})

- {executorsTable} +

+ Executor Summary ({allExecutors.length}) +

+
+ {executorsTable} +
{ if (removedExecutors.nonEmpty) { -

Removed Executors ({removedExecutors.length})

++ - removedExecutorsTable +

+ Removed Executors ({removedExecutors.length}) +

++ +
+ {removedExecutorsTable} +
} }
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index bc0bf6a1d970..2d3433fd0b3d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -128,15 +128,25 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
-

Workers ({workers.length})

- {workerTable} +

+ Workers ({workers.length}) +

+
+ {workerTable} +
-

Running Applications ({activeApps.length})

- {activeAppsTable} +

+ Running Applications ({activeApps.length}) +

+
+ {activeAppsTable} +
@@ -144,8 +154,13 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { {if (hasDrivers) {
-

Running Drivers ({activeDrivers.length})

- {activeDriversTable} +

+ Running Drivers ({activeDrivers.length})

+
+ {activeDriversTable} +
} @@ -154,8 +169,14 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
-

Completed Applications ({completedApps.length})

- {completedAppsTable} +

+ Completed Applications ({completedApps.length}) +

+
+ {completedAppsTable} +
@@ -164,8 +185,14 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { if (hasDrivers) {
-

Completed Drivers ({completedDrivers.length})

- {completedDriversTable} +

+ Completed Drivers ({completedDrivers.length}) +

+
+ {completedDriversTable} +
} diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala index ce84bc4dae32..73a1b9d81b6b 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala @@ -77,24 +77,48 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {
-

Running Executors ({runningExecutors.size})

- {runningExecutorTable} +

+ Running Executors ({runningExecutors.size}) +

+
+ {runningExecutorTable} +
{ if (runningDrivers.nonEmpty) { -

Running Drivers ({runningDrivers.size})

++ - runningDriverTable +

+ Running Drivers ({runningDrivers.size}) +

++ +
+ {runningDriverTable} +
} } { if (finishedExecutors.nonEmpty) { -

Finished Executors ({finishedExecutors.size})

++ - finishedExecutorTable +

+ Finished Executors ({finishedExecutors.size}) +

++ +
+ {finishedExecutorTable} +
} } { if (finishedDrivers.nonEmpty) { -

Finished Drivers ({finishedDrivers.size})

++ - finishedDriverTable +

+ Finished Drivers ({finishedDrivers.size}) +

++ +
+ {finishedDriverTable} +
} }
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala index 43adab7a35d6..d88dd8426e0e 100644 --- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala @@ -48,10 +48,38 @@ private[ui] class EnvironmentPage( classPathHeaders, classPathRow, appEnv.classpathEntries, fixedWidth = true) val content = -

Runtime Information

{runtimeInformationTable} -

Spark Properties

{sparkPropertiesTable} -

System Properties

{systemPropertiesTable} -

Classpath Entries

{classpathEntriesTable} +

+ Runtime Information +

+
+ {runtimeInformationTable} +
+

+ Spark Properties +

+
+ {sparkPropertiesTable} +
+

+ System Properties +

+
+ {systemPropertiesTable} +
+

+ Classpath Entries +

+
+ {classpathEntriesTable} +
UIUtils.headerSparkPage("Environment", content, parent) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 37e3b3b304a6..f6cf66eb8952 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -365,16 +365,34 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We store.executorList(false), startTime) if (shouldShowActiveJobs) { - content ++=

Active Jobs ({activeJobs.size})

++ - activeJobsTable + content ++= +

+ Active Jobs ({activeJobs.size}) +

++ +
+ {activeJobsTable} +
} if (shouldShowCompletedJobs) { - content ++=

Completed Jobs ({completedJobNumStr})

++ - completedJobsTable + content ++= +

+ Completed Jobs ({completedJobNumStr}) +

++ +
+ {completedJobsTable} +
} if (shouldShowFailedJobs) { - content ++=

Failed Jobs ({failedJobs.size})

++ - failedJobsTable + content ++= +

+ Failed Jobs ({failedJobs.size}) +

++ +
+ {failedJobsTable} +
} val helpText = """A job is triggered by an action, like count() or saveAsTextFile().""" + diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala index b1e343451e28..a2d1d8696414 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala @@ -116,26 +116,58 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { var content = summary ++ { if (sc.isDefined && isFairScheduler) { -

Fair Scheduler Pools ({pools.size})

++ poolTable.toNodeSeq +

+ Fair Scheduler Pools ({pools.size}) +

++ +
+ {poolTable.toNodeSeq} +
} else { Seq.empty[Node] } } if (shouldShowActiveStages) { - content ++=

Active Stages ({activeStages.size})

++ - activeStagesTable.toNodeSeq + content ++= +

+ Active Stages ({activeStages.size}) +

++ +
+ {activeStagesTable.toNodeSeq} +
} if (shouldShowPendingStages) { - content ++=

Pending Stages ({pendingStages.size})

++ - pendingStagesTable.toNodeSeq + content ++= +

+ Pending Stages ({pendingStages.size}) +

++ +
+ {pendingStagesTable.toNodeSeq} +
} if (shouldShowCompletedStages) { - content ++=

Completed Stages ({completedStageNumStr})

++ - completedStagesTable.toNodeSeq + content ++= +

+ Completed Stages ({completedStageNumStr}) +

++ +
+ {completedStagesTable.toNodeSeq} +
} if (shouldShowFailedStages) { - content ++=

Failed Stages ({numFailedStages})

++ - failedStagesTable.toNodeSeq + content ++= +

+ Failed Stages ({numFailedStages}) +

++ +
+ {failedStagesTable.toNodeSeq} +
} UIUtils.headerSparkPage("Stages for All Jobs", content, parent) } diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index 740f12e7d13d..c00c0f68fcaa 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -340,24 +340,57 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP jobId, store.operationGraphForJob(jobId)) if (shouldShowActiveStages) { - content ++=

Active Stages ({activeStages.size})

++ - activeStagesTable.toNodeSeq + content ++= +

+ Active Stages ({activeStages.size}) +

++ +
+ {activeStagesTable.toNodeSeq} +
} if (shouldShowPendingStages) { - content ++=

Pending Stages ({pendingOrSkippedStages.size})

++ - pendingOrSkippedStagesTable.toNodeSeq + content ++= +

+ Pending Stages ({pendingOrSkippedStages.size}) +

++ +
+ {pendingOrSkippedStagesTable.toNodeSeq} +
} if (shouldShowCompletedStages) { - content ++=

Completed Stages ({completedStages.size})

++ - completedStagesTable.toNodeSeq + content ++= +

+ Completed Stages ({completedStages.size}) +

++ +
+ {completedStagesTable.toNodeSeq} +
} if (shouldShowSkippedStages) { - content ++=

Skipped Stages ({pendingOrSkippedStages.size})

++ - pendingOrSkippedStagesTable.toNodeSeq + content ++= +

+ Skipped Stages ({pendingOrSkippedStages.size}) +

++ +
+ {pendingOrSkippedStagesTable.toNodeSeq} +
} if (shouldShowFailedStages) { - content ++=

Failed Stages ({failedStages.size})

++ - failedStagesTable.toNodeSeq + content ++= +

+ Failed Stages ({failedStages.size}) +

++ +
+ {failedStagesTable.toNodeSeq} +
} UIUtils.headerSparkPage(s"Details for Job $jobId", content, parent, showVisualization = true) } diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala index 98fbd7aceaa1..29e11b1c0e62 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala @@ -51,7 +51,14 @@ private[ui] class PoolPage(parent: StagesTab) extends WebUIPage("pool") { val poolTable = new PoolTable(Map(pool -> uiPool), parent) var content =

Summary

++ poolTable.toNodeSeq if (activeStages.nonEmpty) { - content ++=

Active Stages ({activeStages.size})

++ activeStagesTable.toNodeSeq + content ++= +

+ Active Stages ({activeStages.size}) +

++ +
+ {activeStagesTable.toNodeSeq} +
} UIUtils.headerSparkPage("Fair Scheduler Pool: " + poolName, content, parent) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 11a6a3434497..80ec90da1410 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -556,8 +556,13 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
{summaryTable.getOrElse("No tasks have reported metrics yet.")}
++ aggMetrics ++ maybeAccumulableTable ++ -

Tasks ({totalTasksNumStr})

++ - taskTableHTML ++ jsForScrollingDownToTaskTable +

+ Tasks ({totalTasksNumStr}) +

++ +
+ {taskTableHTML ++ jsForScrollingDownToTaskTable} +
UIUtils.headerSparkPage(stageHeader, content, parent, showVisualization = true) } diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala index b8aec9890247..d2ac12821f95 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala @@ -41,8 +41,13 @@ private[ui] class StoragePage(parent: SparkUITab, store: AppStatusStore) extends Nil } else {
-

RDDs

- {UIUtils.listingTable(rddHeader, rddRow, rdds, id = Some("storage-by-rdd-table"))} +

+ RDDs +

+
+ {UIUtils.listingTable(rddHeader, rddRow, rdds, id = Some("storage-by-rdd-table"))} +
} } From c5e71ccf3eaa33e807018535d634d48a467ca4a7 Mon Sep 17 00:00:00 2001 From: guoxiaolong Date: Fri, 12 Jan 2018 13:37:40 +0800 Subject: [PATCH 2/4] Fix the code, increase the arrow of the form page, maintain the consistency of the function. --- .../spark/deploy/master/ui/ApplicationPage.scala | 6 ++++-- .../spark/deploy/master/ui/MasterPage.scala | 16 +++++++++++----- .../spark/deploy/worker/ui/WorkerPage.scala | 12 ++++++++---- .../apache/spark/ui/env/EnvironmentPage.scala | 12 ++++++++---- .../org/apache/spark/ui/jobs/AllJobsPage.scala | 9 ++++++--- .../org/apache/spark/ui/jobs/AllStagesPage.scala | 15 ++++++++++----- .../scala/org/apache/spark/ui/jobs/JobPage.scala | 15 ++++++++++----- .../org/apache/spark/ui/jobs/PoolPage.scala | 3 ++- .../org/apache/spark/ui/jobs/StagePage.scala | 3 ++- .../apache/spark/ui/storage/StoragePage.scala | 3 ++- 10 files changed, 63 insertions(+), 31 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala index 4d15b690a07d..eade42d0bd7c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala @@ -102,7 +102,8 @@ private[ui] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app")

- Executor Summary ({allExecutors.length}) + + Executor Summary ({allExecutors.length})

{executorsTable} @@ -112,7 +113,8 @@ private[ui] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app")

- Removed Executors ({removedExecutors.length}) + + Removed Executors ({removedExecutors.length})

++
{removedExecutorsTable} diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index 2d3433fd0b3d..db4e52365b2f 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -130,7 +130,8 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {

- Workers ({workers.length}) + + Workers ({workers.length})

{workerTable} @@ -142,7 +143,8 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {

- Running Applications ({activeApps.length}) + + Running Applications ({activeApps.length})

{activeAppsTable} @@ -157,7 +159,9 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {

- Running Drivers ({activeDrivers.length})

+ + Running Drivers ({activeDrivers.length}) +
{activeDriversTable}
@@ -172,7 +176,8 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {

- Completed Applications ({completedApps.length}) + + Completed Applications ({completedApps.length})

{completedAppsTable} @@ -188,7 +193,8 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {

- Completed Drivers ({completedDrivers.length}) + + Completed Drivers ({completedDrivers.length})

{completedDriversTable} diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala index 73a1b9d81b6b..5ee8013eb191 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala @@ -80,7 +80,8 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {

- Running Executors ({runningExecutors.size}) + + Running Executors ({runningExecutors.size})

{runningExecutorTable} @@ -90,7 +91,8 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {

- Running Drivers ({runningDrivers.size}) + + Running Drivers ({runningDrivers.size})

++
{runningDriverTable} @@ -102,7 +104,8 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {

- Finished Executors ({finishedExecutors.size}) + + Finished Executors ({finishedExecutors.size})

++
{finishedExecutorTable} @@ -114,7 +117,8 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {

- Finished Drivers ({finishedDrivers.size}) + + Finished Drivers ({finishedDrivers.size})

++
{finishedDriverTable} diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala index d88dd8426e0e..0eaa6ba96661 100644 --- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala @@ -51,7 +51,8 @@ private[ui] class EnvironmentPage(

- Runtime Information + + Runtime Information

{runtimeInformationTable} @@ -59,7 +60,8 @@ private[ui] class EnvironmentPage(

- Spark Properties + + Spark Properties

{sparkPropertiesTable} @@ -67,7 +69,8 @@ private[ui] class EnvironmentPage(

- System Properties + + System Properties

{systemPropertiesTable} @@ -75,7 +78,8 @@ private[ui] class EnvironmentPage(

- Classpath Entries + + Classpath Entries

{classpathEntriesTable} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index f6cf66eb8952..936972ba456a 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -368,7 +368,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We content ++=

- Active Jobs ({activeJobs.size}) + + Active Jobs ({activeJobs.size})

++
{activeJobsTable} @@ -378,7 +379,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We content ++=

- Completed Jobs ({completedJobNumStr}) + + Completed Jobs ({completedJobNumStr})

++
{completedJobsTable} @@ -388,7 +390,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We content ++=

- Failed Jobs ({failedJobs.size}) + + Failed Jobs ({failedJobs.size})

++
{failedJobsTable} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala index a2d1d8696414..ed51ba35f944 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala @@ -118,7 +118,8 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { if (sc.isDefined && isFairScheduler) {

- Fair Scheduler Pools ({pools.size}) + + Fair Scheduler Pools ({pools.size})

++
{poolTable.toNodeSeq} @@ -131,7 +132,8 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { content ++=

- Active Stages ({activeStages.size}) + + Active Stages ({activeStages.size})

++
{activeStagesTable.toNodeSeq} @@ -141,7 +143,8 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { content ++=

- Pending Stages ({pendingStages.size}) + + Pending Stages ({pendingStages.size})

++
{pendingStagesTable.toNodeSeq} @@ -152,7 +155,8 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {

- Completed Stages ({completedStageNumStr}) + + Completed Stages ({completedStageNumStr})

++
{completedStagesTable.toNodeSeq} @@ -163,7 +167,8 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {

- Failed Stages ({numFailedStages}) + + Failed Stages ({numFailedStages})

++
{failedStagesTable.toNodeSeq} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index c00c0f68fcaa..e21508f2870d 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -343,7 +343,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP content ++=

- Active Stages ({activeStages.size}) + + Active Stages ({activeStages.size})

++
{activeStagesTable.toNodeSeq} @@ -354,7 +355,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP

- Pending Stages ({pendingOrSkippedStages.size}) + + Pending Stages ({pendingOrSkippedStages.size})

++
{pendingOrSkippedStagesTable.toNodeSeq} @@ -365,7 +367,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP

- Completed Stages ({completedStages.size}) + + Completed Stages ({completedStages.size})

++
{completedStagesTable.toNodeSeq} @@ -376,7 +379,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP

- Skipped Stages ({pendingOrSkippedStages.size}) + + Skipped Stages ({pendingOrSkippedStages.size})

++
{pendingOrSkippedStagesTable.toNodeSeq} @@ -386,7 +390,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP content ++=

- Failed Stages ({failedStages.size}) + + Failed Stages ({failedStages.size})

++
{failedStagesTable.toNodeSeq} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala index 29e11b1c0e62..ec7d243594d0 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala @@ -54,7 +54,8 @@ private[ui] class PoolPage(parent: StagesTab) extends WebUIPage("pool") { content ++=

- Active Stages ({activeStages.size}) + + Active Stages ({activeStages.size})

++
{activeStagesTable.toNodeSeq} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 80ec90da1410..264f4232b641 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -558,7 +558,8 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We maybeAccumulableTable ++

- Tasks ({totalTasksNumStr}) + + Tasks ({totalTasksNumStr})

++
{taskTableHTML ++ jsForScrollingDownToTaskTable} diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala index d2ac12821f95..37c8afa9c6d0 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala @@ -43,7 +43,8 @@ private[ui] class StoragePage(parent: SparkUITab, store: AppStatusStore) extends

- RDDs + + RDDs

{UIUtils.listingTable(rddHeader, rddRow, rdds, id = Some("storage-by-rdd-table"))} From 1e3df815d2b647373bcca7dda47ff894525e1c2a Mon Sep 17 00:00:00 2001 From: guoxiaolong Date: Mon, 15 Jan 2018 15:04:21 +0800 Subject: [PATCH 3/4] [SPARK-23024]fix the code to fix the style --- .../deploy/master/ui/ApplicationPage.scala | 20 +++++--- .../spark/deploy/master/ui/MasterPage.scala | 50 +++++++++++-------- .../spark/deploy/worker/ui/WorkerPage.scala | 40 +++++++++------ .../apache/spark/ui/env/EnvironmentPage.scala | 40 +++++++++------ .../apache/spark/ui/jobs/AllJobsPage.scala | 30 ++++++----- .../apache/spark/ui/jobs/AllStagesPage.scala | 50 +++++++++++-------- .../org/apache/spark/ui/jobs/JobPage.scala | 50 +++++++++++-------- .../org/apache/spark/ui/jobs/PoolPage.scala | 10 ++-- .../org/apache/spark/ui/jobs/StagePage.scala | 10 ++-- .../apache/spark/ui/storage/StoragePage.scala | 10 ++-- 10 files changed, 186 insertions(+), 124 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala index eade42d0bd7c..f699c75085fe 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala @@ -100,22 +100,26 @@ private[ui] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app")
-

- - Executor Summary ({allExecutors.length}) -

+

+ + Executor Summary ({allExecutors.length}) +

+
{executorsTable}
{ if (removedExecutors.nonEmpty) { -

- - Removed Executors ({removedExecutors.length}) -

++ +

+ + Removed Executors ({removedExecutors.length}) +

+ ++
{removedExecutorsTable}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala index db4e52365b2f..c629937606b5 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala @@ -128,11 +128,13 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
-

- - Workers ({workers.length}) -

+

+ + Workers ({workers.length}) +

+
{workerTable}
@@ -141,11 +143,13 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
-

- - Running Applications ({activeApps.length}) -

+

+ + Running Applications ({activeApps.length}) +

+
{activeAppsTable}
@@ -156,12 +160,14 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { {if (hasDrivers) {
-

- - Running Drivers ({activeDrivers.length}) -

+

+ + Running Drivers ({activeDrivers.length}) +

+
{activeDriversTable}
@@ -173,12 +179,14 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
-

- - Completed Applications ({completedApps.length}) -

+

+ + Completed Applications ({completedApps.length}) +

+
{completedAppsTable}
@@ -190,12 +198,14 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") { if (hasDrivers) {
-

- - Completed Drivers ({completedDrivers.length}) -

+

+ + Completed Drivers ({completedDrivers.length}) +

+
{completedDriversTable}
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala index 5ee8013eb191..8b98ae56fc10 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala @@ -77,23 +77,27 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") {
-

- - Running Executors ({runningExecutors.size}) -

+

+ + Running Executors ({runningExecutors.size}) +

+
{runningExecutorTable}
{ if (runningDrivers.nonEmpty) { -

- - Running Drivers ({runningDrivers.size}) -

++ +

+ + Running Drivers ({runningDrivers.size}) +

+ ++
{runningDriverTable}
@@ -101,12 +105,14 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") { } { if (finishedExecutors.nonEmpty) { -

- - Finished Executors ({finishedExecutors.size}) -

++ +

+ + Finished Executors ({finishedExecutors.size}) +

+ ++
{finishedExecutorTable}
@@ -114,12 +120,14 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends WebUIPage("") { } { if (finishedDrivers.nonEmpty) { -

- - Finished Drivers ({finishedDrivers.size}) -

++ +

+ + Finished Drivers ({finishedDrivers.size}) +

+ ++
{finishedDriverTable}
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala index 0eaa6ba96661..902eb92b854f 100644 --- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala @@ -48,39 +48,47 @@ private[ui] class EnvironmentPage( classPathHeaders, classPathRow, appEnv.classpathEntries, fixedWidth = true) val content = -

- - Runtime Information -

+

+ + Runtime Information +

+
{runtimeInformationTable}
-

- - Spark Properties -

+

+ + Spark Properties +

+
{sparkPropertiesTable}
-

- - System Properties -

+

+ + System Properties +

+
{systemPropertiesTable}
-

- - Classpath Entries -

+

+ + Classpath Entries +

+
{classpathEntriesTable}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 936972ba456a..6b2a3812b190 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -366,33 +366,39 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We if (shouldShowActiveJobs) { content ++= -

- - Active Jobs ({activeJobs.size}) -

++ +

+ + Active Jobs ({activeJobs.size}) +

+ ++
{activeJobsTable}
} if (shouldShowCompletedJobs) { content ++= -

- - Completed Jobs ({completedJobNumStr}) -

++ +

+ + Completed Jobs ({completedJobNumStr}) +

+ ++
{completedJobsTable}
} if (shouldShowFailedJobs) { content ++= -

- - Failed Jobs ({failedJobs.size}) -

++ +

+ + Failed Jobs ({failedJobs.size}) +

+ ++
{failedJobsTable}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala index ed51ba35f944..da1e3224e657 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala @@ -116,11 +116,13 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { var content = summary ++ { if (sc.isDefined && isFairScheduler) { -

- - Fair Scheduler Pools ({pools.size}) -

++ +

+ + Fair Scheduler Pools ({pools.size}) +

+ ++
{poolTable.toNodeSeq}
@@ -130,46 +132,54 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { } if (shouldShowActiveStages) { content ++= -

- - Active Stages ({activeStages.size}) -

++ +

+ + Active Stages ({activeStages.size}) +

+ ++
{activeStagesTable.toNodeSeq}
} if (shouldShowPendingStages) { content ++= -

- - Pending Stages ({pendingStages.size}) -

++ +

+ + Pending Stages ({pendingStages.size}) +

+ ++
{pendingStagesTable.toNodeSeq}
} if (shouldShowCompletedStages) { content ++= -

- - Completed Stages ({completedStageNumStr}) -

++ +

+ + Completed Stages ({completedStageNumStr}) +

+ ++
{completedStagesTable.toNodeSeq}
} if (shouldShowFailedStages) { content ++= -

- - Failed Stages ({numFailedStages}) -

++ +

+ + Failed Stages ({numFailedStages}) +

+ ++
{failedStagesTable.toNodeSeq}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala index e21508f2870d..5747df51d2d9 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala @@ -341,58 +341,68 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP if (shouldShowActiveStages) { content ++= -

- - Active Stages ({activeStages.size}) -

++ +

+ + Active Stages ({activeStages.size}) +

+ ++
{activeStagesTable.toNodeSeq}
} if (shouldShowPendingStages) { content ++= -

- - Pending Stages ({pendingOrSkippedStages.size}) -

++ +

+ + Pending Stages ({pendingOrSkippedStages.size}) +

+ ++
{pendingOrSkippedStagesTable.toNodeSeq}
} if (shouldShowCompletedStages) { content ++= -

- - Completed Stages ({completedStages.size}) -

++ +

+ + Completed Stages ({completedStages.size}) +

+ ++
{completedStagesTable.toNodeSeq}
} if (shouldShowSkippedStages) { content ++= -

- - Skipped Stages ({pendingOrSkippedStages.size}) -

++ +

+ + Skipped Stages ({pendingOrSkippedStages.size}) +

+ ++
{pendingOrSkippedStagesTable.toNodeSeq}
} if (shouldShowFailedStages) { content ++= -

- - Failed Stages ({failedStages.size}) -

++ +

+ + Failed Stages ({failedStages.size}) +

+ ++
{failedStagesTable.toNodeSeq}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala index ec7d243594d0..1cf5a5a6913c 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala @@ -52,11 +52,13 @@ private[ui] class PoolPage(parent: StagesTab) extends WebUIPage("pool") { var content =

Summary

++ poolTable.toNodeSeq if (activeStages.nonEmpty) { content ++= -

- - Active Stages ({activeStages.size}) -

++ +

+ + Active Stages ({activeStages.size}) +

+ ++
{activeStagesTable.toNodeSeq}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 264f4232b641..73d867bfbd74 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -556,11 +556,13 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
{summaryTable.getOrElse("No tasks have reported metrics yet.")}
++ aggMetrics ++ maybeAccumulableTable ++ -

- - Tasks ({totalTasksNumStr}) -

++ +

+ + Tasks ({totalTasksNumStr}) +

+ ++
{taskTableHTML ++ jsForScrollingDownToTaskTable}
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala index 37c8afa9c6d0..68d946574a37 100644 --- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala @@ -41,11 +41,13 @@ private[ui] class StoragePage(parent: SparkUITab, store: AppStatusStore) extends Nil } else {
-

- - RDDs -

+

+ + RDDs +

+
{UIUtils.listingTable(rddHeader, rddRow, rdds, id = Some("storage-by-rdd-table"))}
From 4b6cdbb23d308f0418468cfd3740e4058926570e Mon Sep 17 00:00:00 2001 From: guoxiaolong Date: Tue, 16 Jan 2018 14:06:35 +0800 Subject: [PATCH 4/4] [SPARK-23024] fix the code --- .../org/apache/spark/ui/static/webui.js | 30 +++++++++++++++++++ .../apache/spark/ui/jobs/AllStagesPage.scala | 30 ++++++++++--------- .../org/apache/spark/ui/jobs/PoolPage.scala | 7 +++-- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js index 0fa1fcf25f8b..e575c4c78970 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.js +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js @@ -50,4 +50,34 @@ function collapseTable(thisName, table){ // to remember if it's collapsed on each page reload $(function() { collapseTablePageLoad('collapse-aggregated-metrics','aggregated-metrics'); + collapseTablePageLoad('collapse-aggregated-executors','aggregated-executors'); + collapseTablePageLoad('collapse-aggregated-removedExecutors','aggregated-removedExecutors'); + collapseTablePageLoad('collapse-aggregated-workers','aggregated-workers'); + collapseTablePageLoad('collapse-aggregated-activeApps','aggregated-activeApps'); + collapseTablePageLoad('collapse-aggregated-activeDrivers','aggregated-activeDrivers'); + collapseTablePageLoad('collapse-aggregated-completedApps','aggregated-completedApps'); + collapseTablePageLoad('collapse-aggregated-completedDrivers','aggregated-completedDrivers'); + collapseTablePageLoad('collapse-aggregated-runningExecutors','aggregated-runningExecutors'); + collapseTablePageLoad('collapse-aggregated-runningDrivers','aggregated-runningDrivers'); + collapseTablePageLoad('collapse-aggregated-finishedExecutors','aggregated-finishedExecutors'); + collapseTablePageLoad('collapse-aggregated-finishedDrivers','aggregated-finishedDrivers'); + collapseTablePageLoad('collapse-aggregated-runtimeInformation','aggregated-runtimeInformation'); + collapseTablePageLoad('collapse-aggregated-sparkProperties','aggregated-sparkProperties'); + collapseTablePageLoad('collapse-aggregated-systemProperties','aggregated-systemProperties'); + collapseTablePageLoad('collapse-aggregated-classpathEntries','aggregated-classpathEntries'); + collapseTablePageLoad('collapse-aggregated-activeJobs','aggregated-activeJobs'); + collapseTablePageLoad('collapse-aggregated-completedJobs','aggregated-completedJobs'); + collapseTablePageLoad('collapse-aggregated-failedJobs','aggregated-failedJobs'); + collapseTablePageLoad('collapse-aggregated-poolTable','aggregated-poolTable'); + collapseTablePageLoad('collapse-aggregated-allActiveStages','aggregated-allActiveStages'); + collapseTablePageLoad('collapse-aggregated-allPendingStages','aggregated-allPendingStages'); + collapseTablePageLoad('collapse-aggregated-allCompletedStages','aggregated-allCompletedStages'); + collapseTablePageLoad('collapse-aggregated-allFailedStages','aggregated-allFailedStages'); + collapseTablePageLoad('collapse-aggregated-activeStages','aggregated-activeStages'); + collapseTablePageLoad('collapse-aggregated-pendingOrSkippedStages','aggregated-pendingOrSkippedStages'); + collapseTablePageLoad('collapse-aggregated-completedStages','aggregated-completedStages'); + collapseTablePageLoad('collapse-aggregated-failedStages','aggregated-failedStages'); + collapseTablePageLoad('collapse-aggregated-poolActiveStages','aggregated-poolActiveStages'); + collapseTablePageLoad('collapse-aggregated-tasks','aggregated-tasks'); + collapseTablePageLoad('collapse-aggregated-rdds','aggregated-rdds'); }); \ No newline at end of file diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala index da1e3224e657..606dc1e180e5 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala @@ -132,55 +132,57 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") { } if (shouldShowActiveStages) { content ++= - +

Active Stages ({activeStages.size})

++ -
+
{activeStagesTable.toNodeSeq}
} if (shouldShowPendingStages) { content ++= - +

Pending Stages ({pendingStages.size})

++ -
+
{pendingStagesTable.toNodeSeq}
} if (shouldShowCompletedStages) { content ++= - +

Completed Stages ({completedStageNumStr})

++ -
+
{completedStagesTable.toNodeSeq}
} if (shouldShowFailedStages) { content ++= - +

Failed Stages ({numFailedStages})

++ -
+
{failedStagesTable.toNodeSeq}
} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala index 1cf5a5a6913c..a3e1f13782e3 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala @@ -52,14 +52,15 @@ private[ui] class PoolPage(parent: StagesTab) extends WebUIPage("pool") { var content =

Summary

++ poolTable.toNodeSeq if (activeStages.nonEmpty) { content ++= - +

Active Stages ({activeStages.size})

++ -
+
{activeStagesTable.toNodeSeq}
}