diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java index 4a15559e55cb..389028dabd9f 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/BypassMergeSortShuffleWriter.java @@ -39,6 +39,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.TaskContext; import org.apache.spark.executor.ShuffleWriteMetrics; +import org.apache.spark.scheduler.HighlyCompressedMapStatus; import org.apache.spark.scheduler.MapStatus; import org.apache.spark.scheduler.MapStatus$; import org.apache.spark.serializer.Serializer; @@ -75,6 +76,7 @@ final class BypassMergeSortShuffleWriter extends ShuffleWriter { private static final Logger logger = LoggerFactory.getLogger(BypassMergeSortShuffleWriter.class); + private final TaskContext taskContext; private final int fileBufferSize; private final boolean transferToEnabled; private final int numPartitions; @@ -115,6 +117,7 @@ final class BypassMergeSortShuffleWriter extends ShuffleWriter { this.shuffleId = dep.shuffleId(); this.partitioner = dep.partitioner(); this.numPartitions = partitioner.numPartitions(); + this.taskContext = taskContext; this.writeMetrics = taskContext.taskMetrics().shuffleWriteMetrics(); this.serializer = dep.serializer(); this.shuffleBlockResolver = shuffleBlockResolver; @@ -169,6 +172,16 @@ public void write(Iterator> records) throws IOException { } } mapStatus = MapStatus$.MODULE$.apply(blockManager.shuffleServerId(), partitionLengths); + if (mapStatus instanceof HighlyCompressedMapStatus) { + if (logger.isDebugEnabled() && partitionLengths.length > 0) { + Tuple2 tuple = SortShuffleWriter$.MODULE$.genBlocksDistributionStr( + partitionLengths, (HighlyCompressedMapStatus) mapStatus, taskContext); + if (!tuple._1.isEmpty()) { + logger.debug(tuple._1); + } + writeMetrics.incUnderestimatedBlocksSize((Long)(tuple._2)); + } + } } @VisibleForTesting diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java index 8a1771848dee..ef355de5bbb7 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java @@ -19,33 +19,30 @@ import javax.annotation.Nullable; import java.io.*; +import java.lang.Long; import java.nio.channels.FileChannel; import java.util.Iterator; -import scala.Option; -import scala.Product2; +import scala.*; import scala.collection.JavaConverters; import scala.reflect.ClassTag; import scala.reflect.ClassTag$; import com.google.common.annotations.VisibleForTesting; -import com.google.common.io.ByteStreams; -import com.google.common.io.Closeables; -import com.google.common.io.Files; +import com.google.common.io.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.io.output.CloseShieldOutputStream; +import org.apache.commons.io.output.CountingOutputStream; import org.apache.spark.*; import org.apache.spark.annotation.Private; import org.apache.spark.executor.ShuffleWriteMetrics; import org.apache.spark.io.CompressionCodec; import org.apache.spark.io.CompressionCodec$; -import org.apache.commons.io.output.CloseShieldOutputStream; -import org.apache.commons.io.output.CountingOutputStream; import org.apache.spark.memory.TaskMemoryManager; import org.apache.spark.network.util.LimitedInputStream; -import org.apache.spark.scheduler.MapStatus; -import org.apache.spark.scheduler.MapStatus$; +import org.apache.spark.scheduler.*; import org.apache.spark.serializer.SerializationStream; import org.apache.spark.serializer.SerializerInstance; import org.apache.spark.shuffle.IndexShuffleBlockResolver; @@ -228,6 +225,16 @@ void closeAndWriteOutput() throws IOException { } } mapStatus = MapStatus$.MODULE$.apply(blockManager.shuffleServerId(), partitionLengths); + if (mapStatus instanceof HighlyCompressedMapStatus) { + if (logger.isDebugEnabled() && partitionLengths.length > 0) { + Tuple2 tuple = SortShuffleWriter$.MODULE$.genBlocksDistributionStr( + partitionLengths, (HighlyCompressedMapStatus) mapStatus, taskContext); + if (!tuple._1.isEmpty()) { + logger.debug(tuple._1); + } + writeMetrics.incUnderestimatedBlocksSize((Long)(tuple._2)); + } + } } @VisibleForTesting diff --git a/core/src/main/scala/org/apache/spark/InternalAccumulator.scala b/core/src/main/scala/org/apache/spark/InternalAccumulator.scala index 82d3098e2e05..9144957f24cc 100644 --- a/core/src/main/scala/org/apache/spark/InternalAccumulator.scala +++ b/core/src/main/scala/org/apache/spark/InternalAccumulator.scala @@ -50,6 +50,7 @@ private[spark] object InternalAccumulator { val REMOTE_BLOCKS_FETCHED = SHUFFLE_READ_METRICS_PREFIX + "remoteBlocksFetched" val LOCAL_BLOCKS_FETCHED = SHUFFLE_READ_METRICS_PREFIX + "localBlocksFetched" val REMOTE_BYTES_READ = SHUFFLE_READ_METRICS_PREFIX + "remoteBytesRead" + val REMOTE_BYTES_READ_TO_MEM = SHUFFLE_READ_METRICS_PREFIX + "remoteBytesReadToMem" val LOCAL_BYTES_READ = SHUFFLE_READ_METRICS_PREFIX + "localBytesRead" val FETCH_WAIT_TIME = SHUFFLE_READ_METRICS_PREFIX + "fetchWaitTime" val RECORDS_READ = SHUFFLE_READ_METRICS_PREFIX + "recordsRead" @@ -60,6 +61,7 @@ private[spark] object InternalAccumulator { val BYTES_WRITTEN = SHUFFLE_WRITE_METRICS_PREFIX + "bytesWritten" val RECORDS_WRITTEN = SHUFFLE_WRITE_METRICS_PREFIX + "recordsWritten" val WRITE_TIME = SHUFFLE_WRITE_METRICS_PREFIX + "writeTime" + val UNDERESTIMATED_BLOCKS_SIZE = SHUFFLE_WRITE_METRICS_PREFIX + "underestimatedBlocksSize" } // Names of output metrics diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala index 8dd1a1ea059b..8e5a8a446ad4 100644 --- a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala @@ -31,6 +31,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { private[executor] val _remoteBlocksFetched = new LongAccumulator private[executor] val _localBlocksFetched = new LongAccumulator private[executor] val _remoteBytesRead = new LongAccumulator + private[executor] val _remoteBytesReadToMem = new LongAccumulator private[executor] val _localBytesRead = new LongAccumulator private[executor] val _fetchWaitTime = new LongAccumulator private[executor] val _recordsRead = new LongAccumulator @@ -50,6 +51,11 @@ class ShuffleReadMetrics private[spark] () extends Serializable { */ def remoteBytesRead: Long = _remoteBytesRead.sum + /** + * Total number of remotes bytes read to memory from the shuffle by this task. + */ + def remoteBytesReadToMem: Long = _remoteBytesReadToMem.sum + /** * Shuffle data that was read from the local disk (as opposed to from a remote executor). */ @@ -80,6 +86,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { private[spark] def incRemoteBlocksFetched(v: Long): Unit = _remoteBlocksFetched.add(v) private[spark] def incLocalBlocksFetched(v: Long): Unit = _localBlocksFetched.add(v) private[spark] def incRemoteBytesRead(v: Long): Unit = _remoteBytesRead.add(v) + private[spark] def incRemoteBytesReadToMem(v: Long): Unit = _remoteBytesReadToMem.add(v) private[spark] def incLocalBytesRead(v: Long): Unit = _localBytesRead.add(v) private[spark] def incFetchWaitTime(v: Long): Unit = _fetchWaitTime.add(v) private[spark] def incRecordsRead(v: Long): Unit = _recordsRead.add(v) @@ -87,6 +94,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { private[spark] def setRemoteBlocksFetched(v: Int): Unit = _remoteBlocksFetched.setValue(v) private[spark] def setLocalBlocksFetched(v: Int): Unit = _localBlocksFetched.setValue(v) private[spark] def setRemoteBytesRead(v: Long): Unit = _remoteBytesRead.setValue(v) + private[spark] def setRemoteBytesReadToMem(v: Long): Unit = _remoteBytesReadToMem.setValue(v) private[spark] def setLocalBytesRead(v: Long): Unit = _localBytesRead.setValue(v) private[spark] def setFetchWaitTime(v: Long): Unit = _fetchWaitTime.setValue(v) private[spark] def setRecordsRead(v: Long): Unit = _recordsRead.setValue(v) @@ -99,6 +107,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { _remoteBlocksFetched.setValue(0) _localBlocksFetched.setValue(0) _remoteBytesRead.setValue(0) + _remoteBytesReadToMem.setValue(0) _localBytesRead.setValue(0) _fetchWaitTime.setValue(0) _recordsRead.setValue(0) @@ -106,6 +115,7 @@ class ShuffleReadMetrics private[spark] () extends Serializable { _remoteBlocksFetched.add(metric.remoteBlocksFetched) _localBlocksFetched.add(metric.localBlocksFetched) _remoteBytesRead.add(metric.remoteBytesRead) + _remoteBytesReadToMem.add(metric.remoteBytesReadToMem) _localBytesRead.add(metric.localBytesRead) _fetchWaitTime.add(metric.fetchWaitTime) _recordsRead.add(metric.recordsRead) @@ -122,6 +132,7 @@ private[spark] class TempShuffleReadMetrics { private[this] var _remoteBlocksFetched = 0L private[this] var _localBlocksFetched = 0L private[this] var _remoteBytesRead = 0L + private[this] var _remoteBytesReadToMem = 0L private[this] var _localBytesRead = 0L private[this] var _fetchWaitTime = 0L private[this] var _recordsRead = 0L @@ -129,6 +140,7 @@ private[spark] class TempShuffleReadMetrics { def incRemoteBlocksFetched(v: Long): Unit = _remoteBlocksFetched += v def incLocalBlocksFetched(v: Long): Unit = _localBlocksFetched += v def incRemoteBytesRead(v: Long): Unit = _remoteBytesRead += v + def incRemoteBytesReadToMem(v: Long): Unit = _remoteBytesReadToMem += v def incLocalBytesRead(v: Long): Unit = _localBytesRead += v def incFetchWaitTime(v: Long): Unit = _fetchWaitTime += v def incRecordsRead(v: Long): Unit = _recordsRead += v @@ -136,6 +148,7 @@ private[spark] class TempShuffleReadMetrics { def remoteBlocksFetched: Long = _remoteBlocksFetched def localBlocksFetched: Long = _localBlocksFetched def remoteBytesRead: Long = _remoteBytesRead + def remoteBytesReadToMem: Long = _remoteBytesReadToMem def localBytesRead: Long = _localBytesRead def fetchWaitTime: Long = _fetchWaitTime def recordsRead: Long = _recordsRead diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala index ada2e1bc0859..a24d3e45ee60 100644 --- a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala @@ -31,6 +31,7 @@ class ShuffleWriteMetrics private[spark] () extends Serializable { private[executor] val _bytesWritten = new LongAccumulator private[executor] val _recordsWritten = new LongAccumulator private[executor] val _writeTime = new LongAccumulator + private[executor] val _underestimatedBlocksSize = new LongAccumulator /** * Number of bytes written for the shuffle by this task. @@ -47,6 +48,11 @@ class ShuffleWriteMetrics private[spark] () extends Serializable { */ def writeTime: Long = _writeTime.sum + /** + * The sum of underestimated sizes of blocks in MapStatus. + */ + def underestimatedBlocksSize: Long = _underestimatedBlocksSize.value + private[spark] def incBytesWritten(v: Long): Unit = _bytesWritten.add(v) private[spark] def incRecordsWritten(v: Long): Unit = _recordsWritten.add(v) private[spark] def incWriteTime(v: Long): Unit = _writeTime.add(v) @@ -57,6 +63,10 @@ class ShuffleWriteMetrics private[spark] () extends Serializable { _recordsWritten.setValue(recordsWritten - v) } + private[spark] def incUnderestimatedBlocksSize(v: Long) = { + _underestimatedBlocksSize.add(v) + } + // Legacy methods for backward compatibility. // TODO: remove these once we make this class private. @deprecated("use bytesWritten instead", "2.0.0") diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala index dfd2f818acda..7b00235f5362 100644 --- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala +++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala @@ -215,12 +215,14 @@ class TaskMetrics private[spark] () extends Serializable { shuffleRead.REMOTE_BLOCKS_FETCHED -> shuffleReadMetrics._remoteBlocksFetched, shuffleRead.LOCAL_BLOCKS_FETCHED -> shuffleReadMetrics._localBlocksFetched, shuffleRead.REMOTE_BYTES_READ -> shuffleReadMetrics._remoteBytesRead, + shuffleRead.REMOTE_BYTES_READ_TO_MEM -> shuffleReadMetrics._remoteBytesReadToMem, shuffleRead.LOCAL_BYTES_READ -> shuffleReadMetrics._localBytesRead, shuffleRead.FETCH_WAIT_TIME -> shuffleReadMetrics._fetchWaitTime, shuffleRead.RECORDS_READ -> shuffleReadMetrics._recordsRead, shuffleWrite.BYTES_WRITTEN -> shuffleWriteMetrics._bytesWritten, shuffleWrite.RECORDS_WRITTEN -> shuffleWriteMetrics._recordsWritten, shuffleWrite.WRITE_TIME -> shuffleWriteMetrics._writeTime, + shuffleWrite.UNDERESTIMATED_BLOCKS_SIZE -> shuffleWriteMetrics._underestimatedBlocksSize, input.BYTES_READ -> inputMetrics._bytesRead, input.RECORDS_READ -> inputMetrics._recordsRead, output.BYTES_WRITTEN -> outputMetrics._bytesWritten, diff --git a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala index b2e9a97129f0..246ae8c55f42 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala @@ -164,6 +164,8 @@ private[spark] class HighlyCompressedMapStatus private ( emptyBlocks.readExternal(in) avgSize = in.readLong() } + + def getAvgSize: Long = avgSize } private[spark] object HighlyCompressedMapStatus { diff --git a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala index 636b88e792bf..6ad30b46504f 100644 --- a/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala +++ b/core/src/main/scala/org/apache/spark/shuffle/sort/SortShuffleWriter.scala @@ -19,10 +19,10 @@ package org.apache.spark.shuffle.sort import org.apache.spark._ import org.apache.spark.internal.Logging -import org.apache.spark.scheduler.MapStatus +import org.apache.spark.scheduler.{HighlyCompressedMapStatus, MapStatus} import org.apache.spark.shuffle.{BaseShuffleHandle, IndexShuffleBlockResolver, ShuffleWriter} import org.apache.spark.storage.ShuffleBlockId -import org.apache.spark.util.Utils +import org.apache.spark.util.{Distribution, Utils} import org.apache.spark.util.collection.ExternalSorter private[spark] class SortShuffleWriter[K, V, C]( @@ -72,6 +72,17 @@ private[spark] class SortShuffleWriter[K, V, C]( val partitionLengths = sorter.writePartitionedFile(blockId, tmp) shuffleBlockResolver.writeIndexFileAndCommit(dep.shuffleId, mapId, partitionLengths, tmp) mapStatus = MapStatus(blockManager.shuffleServerId, partitionLengths) + mapStatus match { + case hc: HighlyCompressedMapStatus => + if (log.isDebugEnabled() && partitionLengths.length > 0) { + SortShuffleWriter.genBlocksDistributionStr(partitionLengths, hc, context) match { + case (logStr, underestimatedSize) if logStr.nonEmpty => + logDebug(logStr) + writeMetrics.incUnderestimatedBlocksSize(underestimatedSize) + } + } + case _ => // no-op + } } finally { if (tmp.exists() && !tmp.delete()) { logError(s"Error while deleting temp file ${tmp.getAbsolutePath}") @@ -114,4 +125,19 @@ private[spark] object SortShuffleWriter { dep.partitioner.numPartitions <= bypassMergeThreshold } } + def genBlocksDistributionStr(lens: Array[Long], hc: HighlyCompressedMapStatus, + ctx: TaskContext): (String, Long) = { + // Distribution of sizes in MapStatus. + Distribution(lens.map(_.toDouble)) match { + case Some(distribution) => + val underestimatedLengths = lens.filter(_ > hc.getAvgSize).map(_ - hc.getAvgSize) + val distributionStr = distribution.getQuantiles().mkString(", ") + (s"For task ${ctx.partitionId()}.${ctx.attemptNumber()} in stage ${ctx.stageId()} " + + s"(TID ${ctx.taskAttemptId()}), the block sizes in MapStatus are highly compressed" + + s" (average is ${hc.getAvgSize}, ${underestimatedLengths.length} blocks underestimated," + + s" the size of underestimated is ${underestimatedLengths.sum}), distribution at " + + s"probabilities(0, 0.25, 0.5, 0.75, 1.0) is $distributionStr.", underestimatedLengths.sum) + case None => ("", 0L) + } + } } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala index 1818935392eb..6037c62d3957 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala @@ -200,6 +200,7 @@ private[v1] object AllStagesResource { readBytes = submetricQuantiles(_.totalBytesRead), readRecords = submetricQuantiles(_.recordsRead), remoteBytesRead = submetricQuantiles(_.remoteBytesRead), + remoteBytesReadToMem = submetricQuantiles(_.remoteBytesReadToMem), remoteBlocksFetched = submetricQuantiles(_.remoteBlocksFetched), localBlocksFetched = submetricQuantiles(_.localBlocksFetched), totalBlocksFetched = submetricQuantiles(_.totalBlocksFetched), @@ -216,7 +217,8 @@ private[v1] object AllStagesResource { def build: ShuffleWriteMetricDistributions = new ShuffleWriteMetricDistributions( writeBytes = submetricQuantiles(_.bytesWritten), writeRecords = submetricQuantiles(_.recordsWritten), - writeTime = submetricQuantiles(_.writeTime) + writeTime = submetricQuantiles(_.writeTime), + underestimatedBlocksSize = submetricQuantiles(_.underestimatedBlocksSize) ) }.build @@ -281,6 +283,7 @@ private[v1] object AllStagesResource { localBlocksFetched = internal.localBlocksFetched, fetchWaitTime = internal.fetchWaitTime, remoteBytesRead = internal.remoteBytesRead, + remoteBytesReadToMem = internal.remoteBytesReadToMem, localBytesRead = internal.localBytesRead, recordsRead = internal.recordsRead ) @@ -290,7 +293,8 @@ private[v1] object AllStagesResource { new ShuffleWriteMetrics( bytesWritten = internal.bytesWritten, writeTime = internal.writeTime, - recordsWritten = internal.recordsWritten + recordsWritten = internal.recordsWritten, + underestimatedBlocksSize = internal.underestimatedBlocksSize ) } } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 5b9227350eda..ebf62bf0d66c 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -196,13 +196,15 @@ class ShuffleReadMetrics private[spark]( val localBlocksFetched: Long, val fetchWaitTime: Long, val remoteBytesRead: Long, + val remoteBytesReadToMem: Long, val localBytesRead: Long, val recordsRead: Long) class ShuffleWriteMetrics private[spark]( val bytesWritten: Long, val writeTime: Long, - val recordsWritten: Long) + val recordsWritten: Long, + val underestimatedBlocksSize: Long) class TaskMetricDistributions private[spark]( val quantiles: IndexedSeq[Double], @@ -237,12 +239,14 @@ class ShuffleReadMetricDistributions private[spark]( val localBlocksFetched: IndexedSeq[Double], val fetchWaitTime: IndexedSeq[Double], val remoteBytesRead: IndexedSeq[Double], + val remoteBytesReadToMem: IndexedSeq[Double], val totalBlocksFetched: IndexedSeq[Double]) class ShuffleWriteMetricDistributions private[spark]( val writeBytes: IndexedSeq[Double], val writeRecords: IndexedSeq[Double], - val writeTime: IndexedSeq[Double]) + val writeTime: IndexedSeq[Double], + val underestimatedBlocksSize: IndexedSeq[Double]) class AccumulableInfo private[spark]( val id: Long, diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala index f8906117638b..737f25a885f1 100644 --- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala +++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala @@ -157,6 +157,7 @@ final class ShuffleBlockFetcherIterator( case SuccessFetchResult(_, address, _, buf, _) => if (address != blockManager.blockManagerId) { shuffleMetrics.incRemoteBytesRead(buf.size) + shuffleMetrics.incRemoteBytesReadToMem(buf.size) shuffleMetrics.incRemoteBlocksFetched(1) } buf.release() @@ -337,6 +338,7 @@ final class ShuffleBlockFetcherIterator( case r @ SuccessFetchResult(blockId, address, size, buf, isNetworkReqDone) => if (address != blockManager.blockManagerId) { shuffleMetrics.incRemoteBytesRead(buf.size) + shuffleMetrics.incRemoteBytesReadToMem(buf.size) shuffleMetrics.incRemoteBlocksFetched(1) } bytesInFlight -= size diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala index 073f7edfc2fe..25c2ab293503 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala @@ -238,6 +238,7 @@ private[spark] object UIData { remoteBlocksFetched: Long, localBlocksFetched: Long, remoteBytesRead: Long, + remoteBytesReadToMem: Long, localBytesRead: Long, fetchWaitTime: Long, recordsRead: Long, @@ -261,6 +262,7 @@ private[spark] object UIData { remoteBlocksFetched = metrics.remoteBlocksFetched, localBlocksFetched = metrics.localBlocksFetched, remoteBytesRead = metrics.remoteBytesRead, + remoteBytesReadToMem = metrics.remoteBytesReadToMem, localBytesRead = metrics.localBytesRead, fetchWaitTime = metrics.fetchWaitTime, recordsRead = metrics.recordsRead, @@ -269,13 +271,14 @@ private[spark] object UIData { ) } } - private val EMPTY = ShuffleReadMetricsUIData(0, 0, 0, 0, 0, 0, 0, 0) + private val EMPTY = ShuffleReadMetricsUIData(0, 0, 0, 0, 0, 0, 0, 0, 0) } case class ShuffleWriteMetricsUIData( bytesWritten: Long, recordsWritten: Long, - writeTime: Long) + writeTime: Long, + underestimatedBlocksSize: Long) object ShuffleWriteMetricsUIData { def apply(metrics: ShuffleWriteMetrics): ShuffleWriteMetricsUIData = { @@ -285,11 +288,12 @@ private[spark] object UIData { new ShuffleWriteMetricsUIData( bytesWritten = metrics.bytesWritten, recordsWritten = metrics.recordsWritten, - writeTime = metrics.writeTime + writeTime = metrics.writeTime, + underestimatedBlocksSize = metrics.underestimatedBlocksSize ) } } - private val EMPTY = ShuffleWriteMetricsUIData(0, 0, 0) + private val EMPTY = ShuffleWriteMetricsUIData(0, 0, 0, 0) } } diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala index 4b4d2d10cbf8..92470cca908c 100644 --- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala +++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala @@ -330,12 +330,15 @@ private[spark] object JsonProtocol { ("Local Blocks Fetched" -> taskMetrics.shuffleReadMetrics.localBlocksFetched) ~ ("Fetch Wait Time" -> taskMetrics.shuffleReadMetrics.fetchWaitTime) ~ ("Remote Bytes Read" -> taskMetrics.shuffleReadMetrics.remoteBytesRead) ~ + ("Remote Bytes Read To Mem" -> taskMetrics.shuffleReadMetrics.remoteBytesReadToMem) ~ ("Local Bytes Read" -> taskMetrics.shuffleReadMetrics.localBytesRead) ~ ("Total Records Read" -> taskMetrics.shuffleReadMetrics.recordsRead) val shuffleWriteMetrics: JValue = ("Shuffle Bytes Written" -> taskMetrics.shuffleWriteMetrics.bytesWritten) ~ ("Shuffle Write Time" -> taskMetrics.shuffleWriteMetrics.writeTime) ~ - ("Shuffle Records Written" -> taskMetrics.shuffleWriteMetrics.recordsWritten) + ("Shuffle Records Written" -> taskMetrics.shuffleWriteMetrics.recordsWritten) ~ + ("Underestimated Blocks Size Writen" -> + taskMetrics.shuffleWriteMetrics.underestimatedBlocksSize) val inputMetrics: JValue = ("Bytes Read" -> taskMetrics.inputMetrics.bytesRead) ~ ("Records Read" -> taskMetrics.inputMetrics.recordsRead) @@ -791,6 +794,8 @@ private[spark] object JsonProtocol { readMetrics.incRemoteBlocksFetched((readJson \ "Remote Blocks Fetched").extract[Int]) readMetrics.incLocalBlocksFetched((readJson \ "Local Blocks Fetched").extract[Int]) readMetrics.incRemoteBytesRead((readJson \ "Remote Bytes Read").extract[Long]) + Utils.jsonOption(readJson \ "Remote Bytes Read To Mem") + .foreach { v => readMetrics.incRemoteBytesReadToMem(v.extract[Long])} readMetrics.incLocalBytesRead( Utils.jsonOption(readJson \ "Local Bytes Read").map(_.extract[Long]).getOrElse(0L)) readMetrics.incFetchWaitTime((readJson \ "Fetch Wait Time").extract[Long]) @@ -807,6 +812,8 @@ private[spark] object JsonProtocol { writeMetrics.incRecordsWritten( Utils.jsonOption(writeJson \ "Shuffle Records Written").map(_.extract[Long]).getOrElse(0L)) writeMetrics.incWriteTime((writeJson \ "Shuffle Write Time").extract[Long]) + Utils.jsonOption(writeJson \ "Underestimated Blocks Size Writen") + .foreach { v => writeMetrics.incUnderestimatedBlocksSize(v.extract[Long])} } // Output metrics diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index c2f450ba87c6..7c7b4cb31079 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -60,13 +60,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 94000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -105,13 +107,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 98000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -150,13 +154,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 76000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -195,13 +201,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1647, "writeTime" : 83000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -240,13 +248,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1645, "writeTime" : 101000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -285,13 +295,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 73000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -330,13 +342,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 88000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -375,13 +389,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 79000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 506859ae545b..ba70852743bc 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -60,13 +60,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 94000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -105,13 +107,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 98000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -150,13 +154,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 76000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -195,13 +201,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1647, "writeTime" : 83000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -240,13 +248,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1645, "writeTime" : 101000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -285,13 +295,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 73000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -330,13 +342,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 88000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -375,13 +389,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1648, "writeTime" : 79000, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json index f4cec68fbfdf..10bf32aeba55 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json @@ -33,13 +33,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3842811, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -77,13 +79,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3934399, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -121,13 +125,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89885, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -165,13 +171,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 1311694, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -209,13 +217,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 83022, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -253,13 +263,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3675510, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -297,13 +309,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 4016617, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -341,13 +355,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 2579051, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -385,13 +401,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 121551, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -429,13 +447,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 101664, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -473,13 +493,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94709, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -517,13 +539,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94507, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -561,13 +585,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102476, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -605,13 +631,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95004, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -649,13 +677,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95646, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -693,13 +723,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 602780, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -737,13 +769,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108320, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -781,13 +815,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 99944, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -825,13 +861,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100836, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -869,13 +907,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95788, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json index 496a21c328da..4c3af6064fbb 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json @@ -38,13 +38,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -87,13 +89,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -136,13 +140,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -185,13 +191,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -234,13 +242,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -283,13 +293,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -332,13 +344,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -381,13 +395,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json index 4328dc753c5d..8d01b10a30f4 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json @@ -38,13 +38,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -87,13 +89,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -136,13 +140,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -185,13 +191,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -234,13 +242,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -283,13 +293,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -332,13 +344,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, { @@ -381,13 +395,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json index 8c571430f3a1..6b5424b86773 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json @@ -33,13 +33,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94709, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -77,13 +79,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94507, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -121,13 +125,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102476, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -165,13 +171,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95004, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -209,13 +217,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95646, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -253,13 +263,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 602780, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -297,13 +309,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108320, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -341,13 +355,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 99944, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -385,13 +401,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100836, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -429,13 +447,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95788, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -473,13 +493,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 97716, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -517,13 +539,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100270, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -561,13 +585,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 143427, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -605,13 +631,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 91844, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -649,13 +677,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 157194, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -693,13 +723,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94134, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -737,13 +769,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108213, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -781,13 +815,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102019, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -825,13 +861,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 104299, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -869,13 +907,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 114938, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -913,13 +953,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 119770, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -957,13 +999,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 92619, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1001,13 +1045,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89603, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1045,13 +1091,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 118329, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1089,13 +1137,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 127746, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1133,13 +1183,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 160963, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1177,13 +1229,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 123855, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1221,13 +1275,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 111869, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1265,13 +1321,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 131158, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1309,13 +1367,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 98748, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1353,13 +1413,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94792, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1397,13 +1459,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 90765, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1441,13 +1505,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 103713, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1485,13 +1551,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 171516, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1529,13 +1597,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 98293, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1573,13 +1643,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 92985, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1617,13 +1689,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 113322, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1661,13 +1735,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 103015, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1705,13 +1781,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 139844, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1749,13 +1827,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94984, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1793,13 +1873,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 90836, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1837,13 +1919,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 96013, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1881,13 +1965,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89664, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1925,13 +2011,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 92835, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -1969,13 +2057,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 90506, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -2013,13 +2103,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108309, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -2057,13 +2149,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 90329, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -2101,13 +2195,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 96849, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -2145,13 +2241,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 97521, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -2189,13 +2287,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100753, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json index 0bd614bdc756..334eb6dd61dc 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json @@ -33,13 +33,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 4016617, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -77,13 +79,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3934399, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -121,13 +125,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3675510, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -165,13 +171,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3842811, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -209,13 +217,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 1311694, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -253,13 +263,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 83022, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -297,13 +309,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 2579051, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -341,13 +355,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89885, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -385,13 +401,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 143427, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -429,13 +447,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100836, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -473,13 +493,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 99944, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -517,13 +539,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100270, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -561,13 +585,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 101664, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -605,13 +631,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108320, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -649,13 +677,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95788, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -693,13 +723,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95646, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -737,13 +769,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 97716, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -781,13 +815,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 121551, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -825,13 +861,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102476, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -869,13 +907,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95004, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json index 0bd614bdc756..334eb6dd61dc 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json @@ -33,13 +33,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 4016617, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -77,13 +79,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3934399, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -121,13 +125,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3675510, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -165,13 +171,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 3842811, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -209,13 +217,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 1311694, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -253,13 +263,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 83022, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -297,13 +309,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 2579051, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -341,13 +355,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89885, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -385,13 +401,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 143427, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -429,13 +447,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100836, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -473,13 +493,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 99944, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -517,13 +539,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100270, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -561,13 +585,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 101664, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -605,13 +631,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 108320, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -649,13 +677,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95788, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -693,13 +723,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95646, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -737,13 +769,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 97716, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -781,13 +815,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 121551, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -825,13 +861,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102476, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -869,13 +907,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95004, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json index b58f1a51ba48..e2c3ffbf6a1e 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json @@ -33,13 +33,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 94792, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -77,13 +79,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 90765, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -121,13 +125,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 171516, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -165,13 +171,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 96849, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -209,13 +217,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 97521, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -253,13 +263,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 101750, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -297,13 +309,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 95848, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -341,13 +355,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89603, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -385,13 +401,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 98748, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -429,13 +447,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 103713, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -473,13 +493,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 96013, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -517,13 +539,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 100753, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -561,13 +585,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102779, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -605,13 +631,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 102159, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -649,13 +677,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 98472, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -693,13 +723,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 133964, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -737,13 +769,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 98293, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -781,13 +815,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 103015, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -825,13 +861,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 90836, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } }, { @@ -869,13 +907,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 1710, "writeTime" : 89664, - "recordsWritten" : 10 + "recordsWritten" : 10, + "underestimatedBlocksSize": 0 } } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json index 0ed609d5b7f9..a6032afa56cb 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json @@ -24,11 +24,13 @@ "localBlocksFetched" : [ 0.0, 0.0, 0.0 ], "fetchWaitTime" : [ 0.0, 0.0, 0.0 ], "remoteBytesRead" : [ 0.0, 0.0, 0.0 ], + "remoteBytesReadToMem" : [ 0.0, 0.0, 0.0 ], "totalBlocksFetched" : [ 0.0, 0.0, 0.0 ] }, "shuffleWriteMetrics" : { "writeBytes" : [ 1710.0, 1710.0, 1710.0 ], "writeRecords" : [ 10.0, 10.0, 10.0 ], - "writeTime" : [ 89437.0, 102159.0, 4016617.0 ] + "writeTime" : [ 89437.0, 102159.0, 4016617.0 ], + "underestimatedBlocksSize": [0.0, 0.0, 0.0] } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json index 6d230ac65377..2700d6de3c1b 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json @@ -24,11 +24,13 @@ "localBlocksFetched" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ], "fetchWaitTime" : [ 0.0, 0.0, 0.0, 1.0, 1.0 ], "remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "remoteBytesReadToMem" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "totalBlocksFetched" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ] }, "shuffleWriteMetrics" : { "writeBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "writeRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], - "writeTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] + "writeTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "underestimatedBlocksSize": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json index aea0f5413d8b..9c46ac302fd1 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json @@ -24,11 +24,13 @@ "localBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "fetchWaitTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "remoteBytesReadToMem" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], "totalBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "shuffleWriteMetrics" : { "writeBytes" : [ 1710.0, 1710.0, 1710.0, 1710.0, 1710.0 ], "writeRecords" : [ 10.0, 10.0, 10.0, 10.0, 10.0 ], - "writeTime" : [ 90329.0, 95848.0, 102159.0, 121551.0, 2579051.0 ] + "writeTime" : [ 90329.0, 95848.0, 102159.0, 121551.0, 2579051.0 ], + "underestimatedBlocksSize": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index a449926ee7dc..bf89496fd0a7 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -69,13 +69,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -119,13 +121,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -169,13 +173,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -219,13 +225,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -269,13 +277,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -319,13 +329,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -369,13 +381,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } }, @@ -419,13 +433,15 @@ "localBlocksFetched" : 0, "fetchWaitTime" : 0, "remoteBytesRead" : 0, + "remoteBytesReadToMem" : 0, "localBytesRead" : 0, "recordsRead" : 0 }, "shuffleWriteMetrics" : { "bytesWritten" : 0, "writeTime" : 0, - "recordsWritten" : 0 + "recordsWritten" : 0, + "underestimatedBlocksSize": 0 } } } diff --git a/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala b/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala index eae26fa742a2..b546bcfb4acd 100644 --- a/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/TaskMetricsSuite.scala @@ -94,6 +94,8 @@ class TaskMetricsSuite extends SparkFunSuite { sr.setRemoteBytesRead(30L) sr.incRemoteBytesRead(3L) sr.incRemoteBytesRead(3L) + sr.setRemoteBytesReadToMem(10L) + sr.incRemoteBytesReadToMem(8L) sr.setLocalBytesRead(400L) sr.setLocalBytesRead(40L) sr.incLocalBytesRead(4L) @@ -110,6 +112,7 @@ class TaskMetricsSuite extends SparkFunSuite { assert(sr.remoteBlocksFetched == 12) assert(sr.localBlocksFetched == 24) assert(sr.remoteBytesRead == 36L) + assert(sr.remoteBytesReadToMem === 18L) assert(sr.localBytesRead == 48L) assert(sr.fetchWaitTime == 60L) assert(sr.recordsRead == 72L) @@ -133,10 +136,12 @@ class TaskMetricsSuite extends SparkFunSuite { sw.decRecordsWritten(2L) sw.incWriteTime(300L) sw.incWriteTime(30L) + sw.incUnderestimatedBlocksSize(1024) // assert new values exist assert(sw.bytesWritten == 108L) assert(sw.recordsWritten == 216L) assert(sw.writeTime == 330L) + assert(sw.underestimatedBlocksSize === 1024) } test("mutating input metrics values") { diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 9f76c74bce89..af0bb6f275ee 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -1127,13 +1127,15 @@ private[spark] object JsonProtocolSuite extends Assertions { | "Local Blocks Fetched": 700, | "Fetch Wait Time": 900, | "Remote Bytes Read": 1000, + | "Remote Bytes Read To Mem": 0, | "Local Bytes Read": 1100, | "Total Records Read": 10 | }, | "Shuffle Write Metrics": { | "Shuffle Bytes Written": 1200, | "Shuffle Write Time": 1500, - | "Shuffle Records Written": 12 + | "Shuffle Records Written": 12, + | "Underestimated Blocks Size Writen" : 0 | }, | "Input Metrics" : { | "Bytes Read" : 0, @@ -1227,13 +1229,15 @@ private[spark] object JsonProtocolSuite extends Assertions { | "Local Blocks Fetched" : 0, | "Fetch Wait Time" : 0, | "Remote Bytes Read" : 0, + | "Remote Bytes Read To Mem" : 0, | "Local Bytes Read" : 0, | "Total Records Read" : 0 | }, | "Shuffle Write Metrics": { | "Shuffle Bytes Written": 1200, | "Shuffle Write Time": 1500, - | "Shuffle Records Written": 12 + | "Shuffle Records Written": 12, + | "Underestimated Blocks Size Writen" : 0 | }, | "Input Metrics": { | "Bytes Read": 2100, @@ -1327,13 +1331,15 @@ private[spark] object JsonProtocolSuite extends Assertions { | "Local Blocks Fetched" : 0, | "Fetch Wait Time" : 0, | "Remote Bytes Read" : 0, + | "Remote Bytes Read To Mem" : 0, | "Local Bytes Read" : 0, | "Total Records Read" : 0 | }, - | "Shuffle Write Metrics" : { + | "Shuffle Write Metrics": { | "Shuffle Bytes Written" : 0, | "Shuffle Write Time" : 0, - | "Shuffle Records Written" : 0 + | "Shuffle Records Written" : 0, + | "Underestimated Blocks Size Writen" : 0 | }, | "Input Metrics": { | "Bytes Read": 2100, @@ -1914,76 +1920,90 @@ private[spark] object JsonProtocolSuite extends Assertions { | }, | { | "ID": 14, - | "Name": "${shuffleRead.LOCAL_BYTES_READ}", + | "Name": "${shuffleRead.REMOTE_BYTES_READ_TO_MEM}", | "Update": 0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 15, - | "Name": "${shuffleRead.FETCH_WAIT_TIME}", + | "Name": "${shuffleRead.LOCAL_BYTES_READ}", | "Update": 0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 16, - | "Name": "${shuffleRead.RECORDS_READ}", + | "Name": "${shuffleRead.FETCH_WAIT_TIME}", | "Update": 0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 17, - | "Name": "${shuffleWrite.BYTES_WRITTEN}", + | "Name": "${shuffleRead.RECORDS_READ}", | "Update": 0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 18, - | "Name": "${shuffleWrite.RECORDS_WRITTEN}", + | "Name": "${shuffleWrite.BYTES_WRITTEN}", | "Update": 0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 19, - | "Name": "${shuffleWrite.WRITE_TIME}", + | "Name": "${shuffleWrite.RECORDS_WRITTEN}", | "Update": 0, | "Internal": true, | "Count Failed Values": true | }, | { | "ID": 20, + | "Name": "${shuffleWrite.WRITE_TIME}", + | "Update": 0, + | "Internal": true, + | "Count Failed Values": true + | }, + | { + | "ID": 21, + | "Name": "${shuffleWrite.UNDERESTIMATED_BLOCKS_SIZE}", + | "Update": 0, + | "Internal": true, + | "Count Failed Values": true + | }, + | { + | "ID": 22, | "Name": "${input.BYTES_READ}", | "Update": 2100, | "Internal": true, | "Count Failed Values": true | }, | { - | "ID": 21, + | "ID": 23, | "Name": "${input.RECORDS_READ}", | "Update": 21, | "Internal": true, | "Count Failed Values": true | }, | { - | "ID": 22, + | "ID": 24, | "Name": "${output.BYTES_WRITTEN}", | "Update": 1200, | "Internal": true, | "Count Failed Values": true | }, | { - | "ID": 23, + | "ID": 25, | "Name": "${output.RECORDS_WRITTEN}", | "Update": 12, | "Internal": true, | "Count Failed Values": true | }, | { - | "ID": 24, + | "ID": 26, | "Name": "$TEST_ACCUM", | "Update": 0, | "Internal": true, diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index bd4528bd2126..56b44af61b94 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -955,6 +955,12 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.classification.RandomForestClassificationModel.setFeatureSubsetStrategy"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.numTrees"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setFeatureSubsetStrategy") + ) ++ Seq( + // [SPARK-19937] Add remote bytes read to memory and underestimated blocks size to metrics. + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.this"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ShuffleReadMetricDistributions.this"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ShuffleWriteMetrics.this"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ShuffleWriteMetricDistributions.this") ) }