From 30752cb9b3e91366fe2ac16ca769e8fc7e8dcf54 Mon Sep 17 00:00:00 2001 From: jisookim Date: Tue, 8 Dec 2015 15:44:39 -0800 Subject: [PATCH 1/8] add cpu time to metrics --- .../scala/org/apache/spark/executor/Executor.scala | 9 +++++++++ .../org/apache/spark/executor/TaskMetrics.scala | 14 ++++++++++++++ .../org/apache/spark/scheduler/JobLogger.scala | 3 ++- .../org/apache/spark/scheduler/ResultTask.scala | 4 ++++ .../apache/spark/scheduler/ShuffleMapTask.scala | 4 ++++ .../scala/org/apache/spark/scheduler/Task.scala | 2 ++ .../spark/status/api/v1/AllStagesResource.scala | 2 ++ .../scala/org/apache/spark/status/api/v1/api.scala | 2 ++ .../apache/spark/ui/jobs/JobProgressListener.scala | 4 ++++ .../scala/org/apache/spark/ui/jobs/UIData.scala | 1 + .../scala/org/apache/spark/util/JsonProtocol.scala | 2 ++ 11 files changed, 46 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 7b68dfe5ad06..1385d4f490aa 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -178,13 +178,16 @@ private[spark] class Executor( } override def run(): Unit = { + val threadMXBean = ManagementFactory.getThreadMXBean val taskMemoryManager = new TaskMemoryManager(env.memoryManager, taskId) val deserializeStartTime = System.currentTimeMillis() + val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L Thread.currentThread.setContextClassLoader(replClassLoader) val ser = env.closureSerializer.newInstance() logInfo(s"Running $taskName (TID $taskId)") execBackend.statusUpdate(taskId, TaskState.RUNNING, EMPTY_BYTE_BUFFER) var taskStart: Long = 0 + var taskStartCpu: Long = 0 startGCTime = computeTotalGcTime() try { @@ -208,6 +211,7 @@ private[spark] class Executor( // Run the actual task and measure its runtime. taskStart = System.currentTimeMillis() + taskStartCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L var threwException = true val (value, accumUpdates) = try { val res = task.run( @@ -228,6 +232,7 @@ private[spark] class Executor( } } val taskFinish = System.currentTimeMillis() + val taskFinishCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L // If the task has been killed, let's fail it. if (task.killed) { @@ -244,8 +249,12 @@ private[spark] class Executor( // includes the Partition. Second, Task.run() deserializes the RDD and function to be run. m.setExecutorDeserializeTime( (taskStart - deserializeStartTime) + task.executorDeserializeTime) + m.setExecutorDeserializeCpuTime( + (taskStartCpu - deserializeStartCpuTime) + task.executorDeserializeCpuTime) // We need to subtract Task.run()'s deserialization time to avoid double-counting m.setExecutorRunTime((taskFinish - taskStart) - task.executorDeserializeTime) + m.setExecutorCpuTime((taskFinishCpu - taskStartCpu) - task.executorDeserializeCpuTime) + m.setJvmGCTime(computeTotalGcTime() - startGCTime) m.setResultSerializationTime(afterSerialization - beforeSerialization) m.updateAccumulators() diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index 42207a955359..86136cbc439c 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -55,6 +55,13 @@ class TaskMetrics extends Serializable { def executorDeserializeTime: Long = _executorDeserializeTime private[spark] def setExecutorDeserializeTime(value: Long) = _executorDeserializeTime = value + /** + * CPU Time taken on the executor to deserialize this task in nanoseconds + */ + private var _executorDeserializeCpuTime: Long = _ + def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime + private[spark] def setExecutorDeserializeCpuTime(value: Long) = _executorDeserializeCpuTime = value + /** * Time the executor spends actually running the task (including fetching shuffle data) @@ -63,6 +70,13 @@ class TaskMetrics extends Serializable { def executorRunTime: Long = _executorRunTime private[spark] def setExecutorRunTime(value: Long) = _executorRunTime = value + /** + * CPU Time the executor spends actually running the task (including fetching shuffle data) in nanoseconds + */ + private var _executorCpuTime: Long = _ + def executorCpuTime: Long = _executorCpuTime + private[spark] def setExecutorCpuTime(value: Long) = _executorCpuTime = value + /** * The number of bytes this task transmitted back to the driver as the TaskResult */ diff --git a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala index f96eb8ca0ae0..b821f00e3ff8 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala @@ -153,6 +153,7 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener " START_TIME=" + taskInfo.launchTime + " FINISH_TIME=" + taskInfo.finishTime + " EXECUTOR_ID=" + taskInfo.executorId + " HOST=" + taskMetrics.hostname val executorRunTime = " EXECUTOR_RUN_TIME=" + taskMetrics.executorRunTime + val executorCpuTime = " EXECUTOR_CPU_TIME=" + taskMetrics.executorCpuTime val gcTime = " GC_TIME=" + taskMetrics.jvmGCTime val inputMetrics = taskMetrics.inputMetrics match { case Some(metrics) => @@ -181,7 +182,7 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener " SHUFFLE_WRITE_TIME=" + metrics.shuffleWriteTime case None => "" } - stageLogInfo(stageId, status + info + executorRunTime + gcTime + inputMetrics + outputMetrics + + stageLogInfo(stageId, status + info + executorRunTime + executorCpuTime + gcTime + inputMetrics + outputMetrics + shuffleReadMetrics + writeMetrics) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index fb693721a9cb..dcd66583f584 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -17,6 +17,7 @@ package org.apache.spark.scheduler +import java.lang.management.ManagementFactory import java.nio.ByteBuffer import java.io._ @@ -56,11 +57,14 @@ private[spark] class ResultTask[T, U]( override def runTask(context: TaskContext): U = { // Deserialize the RDD and the func using the broadcast variables. + val threadMXBean = ManagementFactory.getThreadMXBean val deserializeStartTime = System.currentTimeMillis() + val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)]( ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = System.currentTimeMillis() - deserializeStartTime + _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime else 0L metrics = Some(context.taskMetrics) func(context, rdd.iterator(partition, context)) diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index ea97ef0e746d..84aa79bb1dac 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -17,6 +17,7 @@ package org.apache.spark.scheduler +import java.lang.management.ManagementFactory import java.nio.ByteBuffer import scala.language.existentials @@ -59,11 +60,14 @@ private[spark] class ShuffleMapTask( override def runTask(context: TaskContext): MapStatus = { // Deserialize the RDD using the broadcast variable. + val threadMXBean = ManagementFactory.getThreadMXBean val deserializeStartTime = System.currentTimeMillis() + val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, dep) = ser.deserialize[(RDD[_], ShuffleDependency[_, _, _])]( ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = System.currentTimeMillis() - deserializeStartTime + _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime else 0L metrics = Some(context.taskMetrics) var writer: ShuffleWriter[Any, Any] = null diff --git a/core/src/main/scala/org/apache/spark/scheduler/Task.scala b/core/src/main/scala/org/apache/spark/scheduler/Task.scala index d4bc3a5c900f..733a2a821dc3 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/Task.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/Task.scala @@ -125,6 +125,7 @@ private[spark] abstract class Task[T]( @volatile @transient private var _killed = false protected var _executorDeserializeTime: Long = 0 + protected var _executorDeserializeCpuTime: Long = 0 /** * Whether the task has been killed. @@ -135,6 +136,7 @@ private[spark] abstract class Task[T]( * Returns the amount of time spent deserializing the RDD and function to be run. */ def executorDeserializeTime: Long = _executorDeserializeTime + def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime /** * Kills a task by setting the interrupted flag to true. This relies on the upper level Spark diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala index 31b4dd7c0f42..c0d705a99328 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala @@ -101,6 +101,7 @@ private[v1] object AllStagesResource { numCompleteTasks = stageUiData.numCompleteTasks, numFailedTasks = stageUiData.numFailedTasks, executorRunTime = stageUiData.executorRunTime, + executorCpuTime = stageUiData.executorCpuTime, submissionTime = stageInfo.submissionTime.map(new Date(_)), firstTaskLaunchedTime, completionTime = stageInfo.completionTime.map(new Date(_)), @@ -244,6 +245,7 @@ private[v1] object AllStagesResource { new TaskMetrics( executorDeserializeTime = internal.executorDeserializeTime, executorRunTime = internal.executorRunTime, + executorCpuTime = internal.executorCpuTime, resultSize = internal.resultSize, jvmGcTime = internal.jvmGCTime, resultSerializationTime = internal.resultSerializationTime, diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 5feb1dc2e5b7..c334dfa0c73d 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -120,6 +120,7 @@ class StageData private[spark]( val numFailedTasks: Int, val executorRunTime: Long, + val executorCpuTime: Long, val submissionTime: Option[Date], val firstTaskLaunchedTime: Option[Date], val completionTime: Option[Date], @@ -159,6 +160,7 @@ class TaskData private[spark]( class TaskMetrics private[spark]( val executorDeserializeTime: Long, val executorRunTime: Long, + val executorCpuTime: Long, val resultSize: Long, val jvmGcTime: Long, val resultSerializationTime: Long, diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index ca37829216f2..3faff59bd2d3 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -486,6 +486,10 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { val timeDelta = taskMetrics.executorRunTime - oldMetrics.map(_.executorRunTime).getOrElse(0L) stageData.executorRunTime += timeDelta + + val cpuTimeDelta = + taskMetrics.executorCpuTime - oldMetrics.map(_.executorCpuTime).getOrElse(0L) + stageData.executorCpuTime += cpuTimeDelta } override def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate) { diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala index f008d4018061..2597b85957ba 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala @@ -76,6 +76,7 @@ private[spark] object UIData { var numFailedTasks: Int = _ var executorRunTime: Long = _ + var executorCpuTime: Long = _ var inputBytes: Long = _ var inputRecords: Long = _ diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala index cb0f1bf79f3d..33f467ef4c22 100644 --- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala @@ -310,6 +310,7 @@ private[spark] object JsonProtocol { ("Host Name" -> taskMetrics.hostname) ~ ("Executor Deserialize Time" -> taskMetrics.executorDeserializeTime) ~ ("Executor Run Time" -> taskMetrics.executorRunTime) ~ + ("Executor CPU Time" -> taskMetrics.executorCpuTime) ~ ("Result Size" -> taskMetrics.resultSize) ~ ("JVM GC Time" -> taskMetrics.jvmGCTime) ~ ("Result Serialization Time" -> taskMetrics.resultSerializationTime) ~ @@ -718,6 +719,7 @@ private[spark] object JsonProtocol { metrics.setHostname((json \ "Host Name").extract[String]) metrics.setExecutorDeserializeTime((json \ "Executor Deserialize Time").extract[Long]) metrics.setExecutorRunTime((json \ "Executor Run Time").extract[Long]) + metrics.setExecutorCpuTime((json \ "Executor CPU Time").extract[Long]) metrics.setResultSize((json \ "Result Size").extract[Long]) metrics.setJvmGCTime((json \ "JVM GC Time").extract[Long]) metrics.setResultSerializationTime((json \ "Result Serialization Time").extract[Long]) From 6033aa58d7b684bb8a611c31102005278de19132 Mon Sep 17 00:00:00 2001 From: jisookim Date: Thu, 3 Mar 2016 18:18:28 -0800 Subject: [PATCH 2/8] make it backward-compatible --- .../org/apache/spark/util/JsonProtocol.scala | 5 +- .../complete_stage_list_json_expectation.json | 3 ++ .../failed_stage_list_json_expectation.json | 1 + .../one_stage_attempt_json_expectation.json | 9 ++++ .../one_stage_json_expectation.json | 9 ++++ .../stage_list_json_expectation.json | 4 ++ ...ist_with_accumulable_json_expectation.json | 1 + .../stage_task_list_expectation.json | 20 ++++++++ ...multi_attempt_app_json_1__expectation.json | 8 +++ ...multi_attempt_app_json_2__expectation.json | 8 +++ ...k_list_w__offset___length_expectation.json | 50 +++++++++++++++++++ ...stage_task_list_w__sortBy_expectation.json | 20 ++++++++ ...tBy_short_names___runtime_expectation.json | 20 ++++++++ ...rtBy_short_names__runtime_expectation.json | 20 ++++++++ ...age_with_accumulable_json_expectation.json | 9 ++++ .../apache/spark/util/JsonProtocolSuite.scala | 4 ++ 16 files changed, 190 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala index 33f467ef4c22..72a29178fd7a 100644 --- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala @@ -719,7 +719,10 @@ private[spark] object JsonProtocol { metrics.setHostname((json \ "Host Name").extract[String]) metrics.setExecutorDeserializeTime((json \ "Executor Deserialize Time").extract[Long]) metrics.setExecutorRunTime((json \ "Executor Run Time").extract[Long]) - metrics.setExecutorCpuTime((json \ "Executor CPU Time").extract[Long]) + metrics.setExecutorCpuTime((json \ "Executor CPU Time") match { + case JNothing => 0 + case x => x.extract[Long]} + ) metrics.setResultSize((json \ "Result Size").extract[Long]) metrics.setJvmGCTime((json \ "JVM GC Time").extract[Long]) metrics.setResultSerializationTime((json \ "Result Serialization Time").extract[Long]) diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json index 8f8067f86d57..25c4fff77e0a 100644 --- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 162, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:07.191GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:07.191GMT", "completionTime" : "2015-02-03T16:43:07.226GMT", @@ -31,6 +32,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 3476, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", @@ -56,6 +58,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 4338, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:04.228GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:04.234GMT", "completionTime" : "2015-02-03T16:43:04.819GMT", diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json index 08b692eda802..b86ba1e65de1 100644 --- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 7, "numFailedTasks" : 1, "executorRunTime" : 278, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:06.296GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT", "completionTime" : "2015-02-03T16:43:06.347GMT", diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index b07011d4f113..d76ee42dc681 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 3476, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", @@ -37,6 +38,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 435, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 2, @@ -66,6 +68,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -124,6 +128,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 2, @@ -153,6 +158,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -182,6 +188,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 436, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 0, @@ -211,6 +218,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -240,6 +248,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 435, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 2f71520549e1..51eb20fc4ee7 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 3476, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", @@ -37,6 +38,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 435, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 2, @@ -66,6 +68,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -124,6 +128,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 2, @@ -153,6 +158,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -182,6 +188,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 436, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 0, @@ -211,6 +218,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 434, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, @@ -240,6 +248,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 435, + "executorCpuTime" : 0, "resultSize" : 1902, "jvmGcTime" : 19, "resultSerializationTime" : 1, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json index 5b957ed54955..6509df1508b3 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 162, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:07.191GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:07.191GMT", "completionTime" : "2015-02-03T16:43:07.226GMT", @@ -31,6 +32,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 3476, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:05.829GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:05.829GMT", "completionTime" : "2015-02-03T16:43:06.286GMT", @@ -56,6 +58,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 4338, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:04.228GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:04.234GMT", "completionTime" : "2015-02-03T16:43:04.819GMT", @@ -81,6 +84,7 @@ "numCompleteTasks" : 7, "numFailedTasks" : 1, "executorRunTime" : 278, + "executorCpuTime" : 0, "submissionTime" : "2015-02-03T16:43:06.296GMT", "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT", "completionTime" : "2015-02-03T16:43:06.347GMT", diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json index afa425f8c27b..8496863a9346 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 120, + "executorCpuTime" : 0, "submissionTime" : "2015-03-16T19:25:36.103GMT", "firstTaskLaunchedTime" : "2015-03-16T19:25:36.515GMT", "completionTime" : "2015-03-16T19:25:36.579GMT", diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json index f2cb29b31c85..08380a858ef9 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json @@ -11,6 +11,7 @@ "taskMetrics" : { "executorDeserializeTime" : 32, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -39,6 +40,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 350, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 0, @@ -67,6 +69,7 @@ "taskMetrics" : { "executorDeserializeTime" : 32, "executorRunTime" : 348, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 2, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 2, @@ -123,6 +127,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -151,6 +156,7 @@ "taskMetrics" : { "executorDeserializeTime" : 30, "executorRunTime" : 350, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -179,6 +185,7 @@ "taskMetrics" : { "executorDeserializeTime" : 29, "executorRunTime" : 351, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -207,6 +214,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 0, @@ -235,6 +243,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 80, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -263,6 +272,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -291,6 +301,7 @@ "taskMetrics" : { "executorDeserializeTime" : 8, "executorRunTime" : 73, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -319,6 +330,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 75, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -347,6 +359,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 77, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -375,6 +388,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 76, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -403,6 +417,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -431,6 +446,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 76, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -459,6 +475,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -487,6 +504,7 @@ "taskMetrics" : { "executorDeserializeTime" : 11, "executorRunTime" : 91, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 1, @@ -515,6 +533,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 92, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -543,6 +562,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json index c3febc5fc944..fcae325974ec 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json @@ -16,6 +16,7 @@ "taskMetrics" : { "executorDeserializeTime" : 14, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -40,6 +41,7 @@ "taskMetrics" : { "executorDeserializeTime" : 14, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -64,6 +66,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -88,6 +91,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -112,6 +116,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 1, @@ -136,6 +141,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -160,6 +166,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -184,6 +191,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json index 56d667d88917..bff41984fab5 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json @@ -16,6 +16,7 @@ "taskMetrics" : { "executorDeserializeTime" : 14, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -40,6 +41,7 @@ "taskMetrics" : { "executorDeserializeTime" : 14, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -64,6 +66,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -88,6 +91,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -112,6 +116,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 1, @@ -136,6 +141,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -160,6 +166,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -184,6 +191,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json index e5ec3bc4c712..c9c57ed5049b 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json @@ -11,6 +11,7 @@ "taskMetrics" : { "executorDeserializeTime" : 8, "executorRunTime" : 73, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -39,6 +40,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 75, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -67,6 +69,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 77, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 76, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -123,6 +127,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -151,6 +156,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 76, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -179,6 +185,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -207,6 +214,7 @@ "taskMetrics" : { "executorDeserializeTime" : 11, "executorRunTime" : 91, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 1, @@ -235,6 +243,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 92, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -263,6 +272,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -291,6 +301,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -319,6 +330,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 88, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -347,6 +359,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 93, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -375,6 +388,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 65, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -403,6 +417,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 43, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 1, @@ -431,6 +446,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 49, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -459,6 +475,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 38, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -487,6 +504,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 32, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -515,6 +533,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 29, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -543,6 +562,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 39, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -571,6 +591,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 34, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -599,6 +620,7 @@ "taskMetrics" : { "executorDeserializeTime" : 36, "executorRunTime" : 24, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -627,6 +649,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -655,6 +678,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 43, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -683,6 +707,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 27, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -711,6 +736,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 35, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -739,6 +765,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 29, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -767,6 +794,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 32, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -795,6 +823,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 31, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -823,6 +852,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -851,6 +881,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 14, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -879,6 +910,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -907,6 +939,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -935,6 +968,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -963,6 +997,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -991,6 +1026,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 19, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1019,6 +1055,7 @@ "taskMetrics" : { "executorDeserializeTime" : 1, "executorRunTime" : 31, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 6, "resultSerializationTime" : 0, @@ -1047,6 +1084,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1075,6 +1113,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 24, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 6, "resultSerializationTime" : 0, @@ -1103,6 +1142,7 @@ "taskMetrics" : { "executorDeserializeTime" : 7, "executorRunTime" : 23, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 6, "resultSerializationTime" : 0, @@ -1131,6 +1171,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1159,6 +1200,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1187,6 +1229,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1215,6 +1258,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1243,6 +1287,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1271,6 +1316,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 21, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1299,6 +1345,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 20, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1327,6 +1374,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1355,6 +1403,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -1383,6 +1432,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json index 5657123a2db1..85aed7495973 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json @@ -11,6 +11,7 @@ "taskMetrics" : { "executorDeserializeTime" : 29, "executorRunTime" : 351, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -39,6 +40,7 @@ "taskMetrics" : { "executorDeserializeTime" : 30, "executorRunTime" : 350, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -67,6 +69,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 350, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 0, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -123,6 +127,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 0, @@ -151,6 +156,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 2, @@ -179,6 +185,7 @@ "taskMetrics" : { "executorDeserializeTime" : 32, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -207,6 +214,7 @@ "taskMetrics" : { "executorDeserializeTime" : 32, "executorRunTime" : 348, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 2, @@ -235,6 +243,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 93, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -263,6 +272,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 92, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -291,6 +301,7 @@ "taskMetrics" : { "executorDeserializeTime" : 11, "executorRunTime" : 91, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 1, @@ -319,6 +330,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 88, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -347,6 +359,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -375,6 +388,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -403,6 +417,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -431,6 +446,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -459,6 +475,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -487,6 +504,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 80, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -515,6 +533,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 77, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -543,6 +562,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 76, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json index 5657123a2db1..85aed7495973 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json @@ -11,6 +11,7 @@ "taskMetrics" : { "executorDeserializeTime" : 29, "executorRunTime" : 351, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -39,6 +40,7 @@ "taskMetrics" : { "executorDeserializeTime" : 30, "executorRunTime" : 350, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -67,6 +69,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 350, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 0, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -123,6 +127,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 0, @@ -151,6 +156,7 @@ "taskMetrics" : { "executorDeserializeTime" : 31, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 2, @@ -179,6 +185,7 @@ "taskMetrics" : { "executorDeserializeTime" : 32, "executorRunTime" : 349, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 1, @@ -207,6 +214,7 @@ "taskMetrics" : { "executorDeserializeTime" : 32, "executorRunTime" : 348, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 7, "resultSerializationTime" : 2, @@ -235,6 +243,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 93, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -263,6 +272,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 92, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -291,6 +301,7 @@ "taskMetrics" : { "executorDeserializeTime" : 11, "executorRunTime" : 91, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 1, @@ -319,6 +330,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 88, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -347,6 +359,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -375,6 +388,7 @@ "taskMetrics" : { "executorDeserializeTime" : 5, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -403,6 +417,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 84, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -431,6 +446,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -459,6 +475,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 83, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -487,6 +504,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 80, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -515,6 +533,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 77, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -543,6 +562,7 @@ "taskMetrics" : { "executorDeserializeTime" : 9, "executorRunTime" : 76, + "executorCpuTime" : 0, "resultSize" : 2010, "jvmGcTime" : 0, "resultSerializationTime" : 0, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json index 72fe017e9f85..f8d7f71a594e 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json @@ -11,6 +11,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 14, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -39,6 +40,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 1, @@ -67,6 +69,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -95,6 +98,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -123,6 +127,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -151,6 +156,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -179,6 +185,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 16, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -207,6 +214,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -235,6 +243,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -263,6 +272,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -291,6 +301,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -319,6 +330,7 @@ "taskMetrics" : { "executorDeserializeTime" : 20, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, @@ -347,6 +359,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -375,6 +388,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -403,6 +417,7 @@ "taskMetrics" : { "executorDeserializeTime" : 10, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -431,6 +446,7 @@ "taskMetrics" : { "executorDeserializeTime" : 2, "executorRunTime" : 17, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -459,6 +475,7 @@ "taskMetrics" : { "executorDeserializeTime" : 4, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -487,6 +504,7 @@ "taskMetrics" : { "executorDeserializeTime" : 6, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -515,6 +533,7 @@ "taskMetrics" : { "executorDeserializeTime" : 3, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 0, "resultSerializationTime" : 0, @@ -543,6 +562,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 18, + "executorCpuTime" : 0, "resultSize" : 2065, "jvmGcTime" : 5, "resultSerializationTime" : 0, diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index 12665a152c9e..8d345eae1a6e 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -6,6 +6,7 @@ "numCompleteTasks" : 8, "numFailedTasks" : 0, "executorRunTime" : 120, + "executorCpuTime" : 0, "submissionTime" : "2015-03-16T19:25:36.103GMT", "firstTaskLaunchedTime" : "2015-03-16T19:25:36.515GMT", "completionTime" : "2015-03-16T19:25:36.579GMT", @@ -46,6 +47,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -71,6 +73,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -96,6 +99,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 1, @@ -121,6 +125,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -146,6 +151,7 @@ "taskMetrics" : { "executorDeserializeTime" : 14, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -171,6 +177,7 @@ "taskMetrics" : { "executorDeserializeTime" : 13, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -196,6 +203,7 @@ "taskMetrics" : { "executorDeserializeTime" : 12, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, @@ -221,6 +229,7 @@ "taskMetrics" : { "executorDeserializeTime" : 14, "executorRunTime" : 15, + "executorCpuTime" : 0, "resultSize" : 697, "jvmGcTime" : 0, "resultSerializationTime" : 2, diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 1939ce5c743b..dedc8cf69978 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -1039,6 +1039,7 @@ class JsonProtocolSuite extends SparkFunSuite { | "Host Name": "localhost", | "Executor Deserialize Time": 300, | "Executor Run Time": 400, + | "Executor CPU Time": 0, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, @@ -1128,6 +1129,7 @@ class JsonProtocolSuite extends SparkFunSuite { | "Host Name": "localhost", | "Executor Deserialize Time": 300, | "Executor Run Time": 400, + | "Executor CPU Time": 0, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, @@ -1214,6 +1216,7 @@ class JsonProtocolSuite extends SparkFunSuite { | "Host Name": "localhost", | "Executor Deserialize Time": 300, | "Executor Run Time": 400, + | "Executor CPU Time": 0, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, @@ -1702,6 +1705,7 @@ class JsonProtocolSuite extends SparkFunSuite { | "Host Name": "localhost", | "Executor Deserialize Time": 300, | "Executor Run Time": 400, + | "Executor CPU Time": 0, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, From 7b3844285d52592bbf508cda497bf5640a7ad1b8 Mon Sep 17 00:00:00 2001 From: jisookim Date: Fri, 19 Aug 2016 10:31:55 -0700 Subject: [PATCH 3/8] fix tests --- .../stage_task_summary_w_shuffle_read_expectation.json | 4 ++-- .../stage_task_summary_w_shuffle_write_expectation.json | 4 ++-- .../test/scala/org/apache/spark/util/JsonProtocolSuite.scala | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json index 27901541e74d..6d230ac65377 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json @@ -1,9 +1,9 @@ { "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ], "executorDeserializeTime" : [ 1.0, 2.0, 2.0, 2.0, 3.0 ], - "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0 ], + "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "executorRunTime" : [ 30.0, 74.0, 75.0, 76.0, 79.0 ], - "executorCpuTime" : [ 0.0, 0.0, 0.0 ], + "executorCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "resultSize" : [ 1034.0, 1034.0, 1034.0, 1034.0, 1034.0 ], "jvmGcTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json index e5a54ffea7ab..aea0f5413d8b 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json @@ -1,9 +1,9 @@ { "quantiles" : [ 0.05, 0.25, 0.5, 0.75, 0.95 ], "executorDeserializeTime" : [ 2.0, 2.0, 3.0, 7.0, 31.0 ], - "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0 ], + "executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "executorRunTime" : [ 16.0, 18.0, 28.0, 49.0, 349.0 ], - "executorCpuTime" : [ 0.0, 0.0, 0.0 ], + "executorCpuTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "resultSize" : [ 2010.0, 2065.0, 2065.0, 2065.0, 2065.0 ], "jvmGcTime" : [ 0.0, 0.0, 0.0, 5.0, 7.0 ], "resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 1.0 ], diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 1138c26e03de..bf6869c90b67 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -1810,7 +1810,7 @@ private[spark] object JsonProtocolSuite extends Assertions { | "ID": 3, | "Name": "$EXECUTOR_CPU_TIME", | "Update": 0, - | "Interval": true, + | "Internal": true, | "Count Failed Values": true | }, | { From bd19098e1c08f09af33598b65900db54e8aa07b6 Mon Sep 17 00:00:00 2001 From: jisookim Date: Fri, 19 Aug 2016 10:36:23 -0700 Subject: [PATCH 4/8] remove unnecessary * --- .../scala/org/apache/spark/util/JsonProtocolSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index bf6869c90b67..638d0e42de80 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -498,7 +498,7 @@ private[spark] object JsonProtocolSuite extends Assertions { } /** -------------------------------- * - * | Util methods for comparing events | + | Util methods for comparing events | * --------------------------------- */ private[spark] def assertEquals(event1: SparkListenerEvent, event2: SparkListenerEvent) { @@ -747,7 +747,7 @@ private[spark] object JsonProtocolSuite extends Assertions { /** ----------------------------------- * - * | Util methods for constructing events | + | Util methods for constructing events | * ------------------------------------ */ private val properties = { @@ -857,7 +857,7 @@ private[spark] object JsonProtocolSuite extends Assertions { /** --------------------------------------- * - * | JSON string representation of each event | + | JSON string representation of each event | * ---------------------------------------- */ private val stageSubmittedJsonString = From bf8d4f8ceaf32b56682642132ae5b6efe58e58d5 Mon Sep 17 00:00:00 2001 From: jisookim Date: Fri, 19 Aug 2016 12:51:52 -0700 Subject: [PATCH 5/8] fix scalastyle issue --- .../org/apache/spark/executor/Executor.scala | 15 +++++++++++---- .../org/apache/spark/executor/TaskMetrics.scala | 9 +++++---- .../org/apache/spark/scheduler/ResultTask.scala | 4 +--- .../apache/spark/scheduler/ShuffleMapTask.scala | 8 ++++++-- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 574ec6fb0ff3..395474c8fb75 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -235,7 +235,9 @@ private[spark] class Executor( val threadMXBean = ManagementFactory.getThreadMXBean val taskMemoryManager = new TaskMemoryManager(env.memoryManager, taskId) val deserializeStartTime = System.currentTimeMillis() - val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L + val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L Thread.currentThread.setContextClassLoader(replClassLoader) val ser = env.closureSerializer.newInstance() logInfo(s"Running $taskName (TID $taskId)") @@ -272,7 +274,9 @@ private[spark] class Executor( // Run the actual task and measure its runtime. taskStart = System.currentTimeMillis() - taskStartCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L + taskStartCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L var threwException = true val value = try { val res = task.run( @@ -306,7 +310,9 @@ private[spark] class Executor( } } val taskFinish = System.currentTimeMillis() - val taskFinishCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L + val taskFinishCpu = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L // If the task has been killed, let's fail it. if (task.killed) { @@ -326,7 +332,8 @@ private[spark] class Executor( (taskStartCpu - deserializeStartCpuTime) + task.executorDeserializeCpuTime) // We need to subtract Task.run()'s deserialization time to avoid double-counting task.metrics.setExecutorRunTime((taskFinish - taskStart) - task.executorDeserializeTime) - task.metrics.setExecutorCpuTime((taskFinishCpu - taskStartCpu) - task.executorDeserializeCpuTime) + task.metrics.setExecutorCpuTime( + (taskFinishCpu - taskStartCpu) - task.executorDeserializeCpuTime) task.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime) task.metrics.setResultSerializationTime(afterSerialization - beforeSerialization) diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index d552e0370fec..fb1ad60cc79b 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -61,8 +61,8 @@ class TaskMetrics private[spark] () extends Serializable { def executorDeserializeTime: Long = _executorDeserializeTime.sum /** - * CPU Time taken on the executor to deserialize this task in nanoseconds - */ + * CPU Time taken on the executor to deserialize this task in nanoseconds. + */ def executorDeserializeCpuTime: Long = _executorDeserializeCpuTime.sum /** @@ -71,8 +71,9 @@ class TaskMetrics private[spark] () extends Serializable { def executorRunTime: Long = _executorRunTime.sum /** - * CPU Time the executor spends actually running the task (including fetching shuffle data) in nanoseconds - */ + * CPU Time the executor spends actually running the task + * (including fetching shuffle data) in nanoseconds. + */ def executorCpuTime: Long = _executorCpuTime.sum /** diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala index 8389e5e46028..609f10aee940 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala @@ -17,10 +17,8 @@ package org.apache.spark.scheduler -import java.lang.management.ManagementFactory -import java.nio.ByteBuffer - import java.io._ +import java.lang.management.ManagementFactory import java.nio.ByteBuffer import java.util.Properties diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala index f4af73d78021..448fe02084e0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala @@ -69,12 +69,16 @@ private[spark] class ShuffleMapTask( // Deserialize the RDD using the broadcast variable. val threadMXBean = ManagementFactory.getThreadMXBean val deserializeStartTime = System.currentTimeMillis() - val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime else 0L + val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime + } else 0L val ser = SparkEnv.get.closureSerializer.newInstance() val (rdd, dep) = ser.deserialize[(RDD[_], ShuffleDependency[_, _, _])]( ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader) _executorDeserializeTime = System.currentTimeMillis() - deserializeStartTime - _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime else 0L + _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) { + threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime + } else 0L var writer: ShuffleWriter[Any, Any] = null try { From 5a9ea8e2c2d9562af75baffe5eab735e78f1a8ca Mon Sep 17 00:00:00 2001 From: jisookim Date: Fri, 19 Aug 2016 13:27:16 -0700 Subject: [PATCH 6/8] fix DirectMissingMethodProblem --- project/MimaExcludes.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 688218f6f43a..5d0a99c567e5 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -785,9 +785,13 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.SQLContext.parquetFile"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.SQLContext.applySchema") ) ++ Seq( - // [SPARK-14743] Improve delegation token handling in secure cluster - ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.SparkHadoopUtil.getTimeFromNowToRenewal") - ) + // [SPARK-14743] Improve delegation token handling in secure cluster + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.SparkHadoopUtil.getTimeFromNowToRenewal") + ) ++ Seq( + // [SPARK-12221] Add CPU time to metrics + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskMetrics.this"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.this") + ) } def excludes(version: String) = version match { From 2ff5bdce904f4ca55096ea4616557342003b8043 Mon Sep 17 00:00:00 2001 From: jisookim Date: Tue, 20 Sep 2016 15:37:46 -0700 Subject: [PATCH 7/8] fix mistakes happened during resolving conflicts --- ...ask_list_w__sortBy_short_names__runtime_expectation.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json index 25ea31309b3a..db60ccccbf8c 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json @@ -723,7 +723,7 @@ "speculative" : false, "accumulatorUpdates" : [ ], "taskMetrics" : { - "executorDeserializeTime" : 6, + "executorDeserializeTime" : 2, "executorDeserializeCpuTime" : 0, "executorRunTime" : 18, "executorCpuTime" : 0, @@ -765,7 +765,7 @@ "speculative" : false, "accumulatorUpdates" : [ ], "taskMetrics" : { - "executorDeserializeTime" : 3, + "executorDeserializeTime" : 4, "executorDeserializeCpuTime" : 0, "executorRunTime" : 18, "executorCpuTime" : 0, @@ -792,7 +792,7 @@ }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, - "writeTime" : 98293, + "writeTime" : 90836, "recordsWritten" : 10 } } From f0ef503f9e732b91d405d1e15dade58e78999052 Mon Sep 17 00:00:00 2001 From: jisookim Date: Thu, 22 Sep 2016 22:50:52 -0700 Subject: [PATCH 8/8] fix style and add non-zero cpu times in JsonProtocolSuite testing --- .../org/apache/spark/util/JsonProtocol.scala | 8 ++++---- .../apache/spark/util/JsonProtocolSuite.scala | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala index 7d538186a062..f4fa7b406164 100644 --- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala @@ -763,13 +763,13 @@ private[spark] object JsonProtocol { metrics.setExecutorDeserializeTime((json \ "Executor Deserialize Time").extract[Long]) metrics.setExecutorDeserializeCpuTime((json \ "Executor Deserialize CPU Time") match { case JNothing => 0 - case x => x.extract[Long]} - ) + case x => x.extract[Long] + }) metrics.setExecutorRunTime((json \ "Executor Run Time").extract[Long]) metrics.setExecutorCpuTime((json \ "Executor CPU Time") match { case JNothing => 0 - case x => x.extract[Long]} - ) + case x => x.extract[Long] + }) metrics.setResultSize((json \ "Result Size").extract[Long]) metrics.setJvmGCTime((json \ "JVM GC Time").extract[Long]) metrics.setResultSerializationTime((json \ "Result Serialization Time").extract[Long]) diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 3f13d6677887..d5146d70ebaa 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -819,8 +819,11 @@ private[spark] object JsonProtocolSuite extends Assertions { hasOutput: Boolean, hasRecords: Boolean = true) = { val t = TaskMetrics.empty + // Set CPU times same as wall times for testing purpose t.setExecutorDeserializeTime(a) + t.setExecutorDeserializeCpuTime(a) t.setExecutorRunTime(b) + t.setExecutorCpuTime(b) t.setResultSize(c) t.setJvmGCTime(d) t.setResultSerializationTime(a + b) @@ -1100,9 +1103,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | }, | "Task Metrics": { | "Executor Deserialize Time": 300, - | "Executor Deserialize CPU Time": 0, + | "Executor Deserialize CPU Time": 300, | "Executor Run Time": 400, - | "Executor CPU Time": 0, + | "Executor CPU Time": 400, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, @@ -1200,9 +1203,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | }, | "Task Metrics": { | "Executor Deserialize Time": 300, - | "Executor Deserialize CPU Time": 0, + | "Executor Deserialize CPU Time": 300, | "Executor Run Time": 400, - | "Executor CPU Time": 0, + | "Executor CPU Time": 400, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, @@ -1300,9 +1303,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | }, | "Task Metrics": { | "Executor Deserialize Time": 300, - | "Executor Deserialize CPU Time": 0, + | "Executor Deserialize CPU Time": 300, | "Executor Run Time": 400, - | "Executor CPU Time": 0, + | "Executor CPU Time": 400, | "Result Size": 500, | "JVM GC Time": 600, | "Result Serialization Time": 700, @@ -1795,7 +1798,7 @@ private[spark] object JsonProtocolSuite extends Assertions { | { | "ID": 1, | "Name": "$EXECUTOR_DESERIALIZE_CPU_TIME", - | "Update": 0, + | "Update": 300, | "Internal": true, | "Count Failed Values": true | }, @@ -1810,7 +1813,7 @@ private[spark] object JsonProtocolSuite extends Assertions { | { | "ID": 3, | "Name": "$EXECUTOR_CPU_TIME", - | "Update": 0, + | "Update": 400, | "Internal": true, | "Count Failed Values": true | },