Skip to content

Commit e6091b6

Browse files
author
jinxing
committed
Remove the new metrics from event log
1 parent fea66c5 commit e6091b6

File tree

2 files changed

+4
-80
lines changed

2 files changed

+4
-80
lines changed

‎core/src/main/scala/org/apache/spark/util/JsonProtocol.scala‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -331,26 +331,13 @@ private[spark] object JsonProtocol {
331331
("Local Blocks Fetched" -> taskMetrics.shuffleReadMetrics.localBlocksFetched) ~
332332
("Fetch Wait Time" -> taskMetrics.shuffleReadMetrics.fetchWaitTime) ~
333333
("Remote Bytes Read" -> taskMetrics.shuffleReadMetrics.remoteBytesRead) ~
334-
("Remote Bytes Read To Mem" -> taskMetrics.shuffleReadMetrics.remoteBytesReadToMem) ~
335-
("Remote Bytes Read To Disk" -> taskMetrics.shuffleReadMetrics.remoteBytesReadToDisk) ~
336334
("Local Bytes Read" -> taskMetrics.shuffleReadMetrics.localBytesRead) ~
337335
("Total Records Read" -> taskMetrics.shuffleReadMetrics.recordsRead)
338336

339-
340337
var shuffleWriteMetrics: JValue =
341338
("Shuffle Bytes Written" -> taskMetrics.shuffleWriteMetrics.bytesWritten) ~
342339
("Shuffle Write Time" -> taskMetrics.shuffleWriteMetrics.writeTime) ~
343-
("Shuffle Records Written" -> taskMetrics.shuffleWriteMetrics.recordsWritten) ~
344-
("Shuffle Write Average Block Size" -> taskMetrics.shuffleWriteMetrics.averageBlockSize) ~
345-
("Shuffle Write Underestimated Blocks Num" ->
346-
taskMetrics.shuffleWriteMetrics.underestimatedBlocksNum) ~
347-
("Shuffle Write Underestimated Blocks Size" ->
348-
taskMetrics.shuffleWriteMetrics.underestimatedBlocksSize) merge
349-
taskMetrics.shuffleWriteMetrics.blockSizeDistribution.zipWithIndex.map {
350-
case (size, index) =>
351-
render(s"Shuffle Write Block Size Distribution $index" -> size.asInstanceOf[Long])
352-
}.reduceLeft(_ merge _)
353-
340+
("Shuffle Records Written" -> taskMetrics.shuffleWriteMetrics.recordsWritten)
354341
val inputMetrics: JValue =
355342
("Bytes Read" -> taskMetrics.inputMetrics.bytesRead) ~
356343
("Records Read" -> taskMetrics.inputMetrics.recordsRead)
@@ -806,8 +793,6 @@ private[spark] object JsonProtocol {
806793
readMetrics.incRemoteBlocksFetched((readJson \ "Remote Blocks Fetched").extract[Int])
807794
readMetrics.incLocalBlocksFetched((readJson \ "Local Blocks Fetched").extract[Int])
808795
readMetrics.incRemoteBytesRead((readJson \ "Remote Bytes Read").extract[Long])
809-
readMetrics.incRemoteBytesReadToMem((readJson \ "Remote Bytes Read To Mem").extract[Long])
810-
readMetrics.incRemoteBytesReadToDisk((readJson \ "Remote Bytes Read To Disk").extract[Long])
811796
readMetrics.incLocalBytesRead(
812797
Utils.jsonOption(readJson \ "Local Bytes Read").map(_.extract[Long]).getOrElse(0L))
813798
readMetrics.incFetchWaitTime((readJson \ "Fetch Wait Time").extract[Long])
@@ -824,17 +809,6 @@ private[spark] object JsonProtocol {
824809
writeMetrics.incRecordsWritten(
825810
Utils.jsonOption(writeJson \ "Shuffle Records Written").map(_.extract[Long]).getOrElse(0L))
826811
writeMetrics.incWriteTime((writeJson \ "Shuffle Write Time").extract[Long])
827-
(0 until 9).foreach {
828-
case i =>
829-
writeMetrics.setBlockSizeDistribution(i, (writeJson \
830-
s"Shuffle Write Block Size Distribution $i").extract[Long])
831-
}
832-
writeMetrics.setAverageBlockSize(
833-
(writeJson \ "Shuffle Write Average Block Size").extract[Long])
834-
writeMetrics.setUnderestimatedBlocksNum(
835-
(writeJson \ "Shuffle Write Underestimated Blocks Num").extract[Long])
836-
writeMetrics.incUnderestimatedBlocksSize(
837-
(writeJson \ "Shuffle Write Underestimated Blocks Size").extract[Long])
838812
}
839813

840814
// Output metrics

‎core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala‎

Lines changed: 3 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -847,8 +847,6 @@ private[spark] object JsonProtocolSuite extends Assertions {
847847
} else {
848848
val sr = t.createTempShuffleReadMetrics()
849849
sr.incRemoteBytesRead(b + d)
850-
sr.incRemoteBytesReadToMem(b)
851-
sr.incRemoteBytesReadToMem(d)
852850
sr.incLocalBlocksFetched(e)
853851
sr.incFetchWaitTime(a + d)
854852
sr.incRemoteBlocksFetched(f)
@@ -863,12 +861,6 @@ private[spark] object JsonProtocolSuite extends Assertions {
863861
val sw = t.shuffleWriteMetrics
864862
sw.incBytesWritten(a + b + c)
865863
sw.incWriteTime(b + c + d)
866-
sw.setBlockSizeDistribution(0, a)
867-
sw.setBlockSizeDistribution(1, b)
868-
sw.setBlockSizeDistribution(2, c)
869-
sw.setAverageBlockSize(b)
870-
sw.setUnderestimatedBlocksNum(c)
871-
sw.incUnderestimatedBlocksSize(a + b + c)
872864
sw.incRecordsWritten(if (hasRecords) (a + b + c) / 100 else -1)
873865
}
874866
// Make at most 6 blocks
@@ -1135,27 +1127,13 @@ private[spark] object JsonProtocolSuite extends Assertions {
11351127
| "Local Blocks Fetched": 700,
11361128
| "Fetch Wait Time": 900,
11371129
| "Remote Bytes Read": 1000,
1138-
| "Remote Bytes Read To Mem": 1000,
1139-
| "Remote Bytes Read To Disk": 0,
11401130
| "Local Bytes Read": 1100,
11411131
| "Total Records Read": 10
11421132
| },
11431133
| "Shuffle Write Metrics" : {
11441134
| "Shuffle Bytes Written" : 1200,
11451135
| "Shuffle Write Time" : 1500,
1146-
| "Shuffle Records Written" : 12,
1147-
| "Shuffle Write Average Block Size" : 400,
1148-
| "Shuffle Write Underestimated Blocks Num" : 500,
1149-
| "Shuffle Write Underestimated Blocks Size" : 1200,
1150-
| "Shuffle Write Block Size Distribution 0" : 300,
1151-
| "Shuffle Write Block Size Distribution 1" : 400,
1152-
| "Shuffle Write Block Size Distribution 2" : 500,
1153-
| "Shuffle Write Block Size Distribution 3" : 0,
1154-
| "Shuffle Write Block Size Distribution 4" : 0,
1155-
| "Shuffle Write Block Size Distribution 5" : 0,
1156-
| "Shuffle Write Block Size Distribution 6" : 0,
1157-
| "Shuffle Write Block Size Distribution 7" : 0,
1158-
| "Shuffle Write Block Size Distribution 8" : 0
1136+
| "Shuffle Records Written" : 12
11591137
| },
11601138
| "Input Metrics" : {
11611139
| "Bytes Read" : 0,
@@ -1249,27 +1227,13 @@ private[spark] object JsonProtocolSuite extends Assertions {
12491227
| "Local Blocks Fetched" : 0,
12501228
| "Fetch Wait Time" : 0,
12511229
| "Remote Bytes Read" : 0,
1252-
| "Remote Bytes Read To Mem" : 0,
1253-
| "Remote Bytes Read To Disk" : 0,
12541230
| "Local Bytes Read" : 0,
12551231
| "Total Records Read" : 0
12561232
| },
12571233
| "Shuffle Write Metrics": {
12581234
| "Shuffle Bytes Written": 1200,
12591235
| "Shuffle Write Time": 1500,
1260-
| "Shuffle Records Written": 12,
1261-
| "Shuffle Write Average Block Size" : 400,
1262-
| "Shuffle Write Underestimated Blocks Num" : 500,
1263-
| "Shuffle Write Underestimated Blocks Size" : 1200,
1264-
| "Shuffle Write Block Size Distribution 0" : 300,
1265-
| "Shuffle Write Block Size Distribution 1" : 400,
1266-
| "Shuffle Write Block Size Distribution 2" : 500,
1267-
| "Shuffle Write Block Size Distribution 3" : 0,
1268-
| "Shuffle Write Block Size Distribution 4" : 0,
1269-
| "Shuffle Write Block Size Distribution 5" : 0,
1270-
| "Shuffle Write Block Size Distribution 6" : 0,
1271-
| "Shuffle Write Block Size Distribution 7" : 0,
1272-
| "Shuffle Write Block Size Distribution 8" : 0
1236+
| "Shuffle Records Written": 12
12731237
| },
12741238
| "Input Metrics": {
12751239
| "Bytes Read": 2100,
@@ -1363,27 +1327,13 @@ private[spark] object JsonProtocolSuite extends Assertions {
13631327
| "Local Blocks Fetched" : 0,
13641328
| "Fetch Wait Time" : 0,
13651329
| "Remote Bytes Read" : 0,
1366-
| "Remote Bytes Read To Mem" : 0,
1367-
| "Remote Bytes Read To Disk" : 0,
13681330
| "Local Bytes Read" : 0,
13691331
| "Total Records Read" : 0
13701332
| },
13711333
| "Shuffle Write Metrics": {
13721334
| "Shuffle Bytes Written" : 0,
13731335
| "Shuffle Write Time" : 0,
1374-
| "Shuffle Records Written" : 0,
1375-
| "Shuffle Write Average Block Size" : 0,
1376-
| "Shuffle Write Underestimated Blocks Num" : 0,
1377-
| "Shuffle Write Underestimated Blocks Size" : 0,
1378-
| "Shuffle Write Block Size Distribution 0" : 0,
1379-
| "Shuffle Write Block Size Distribution 1" : 0,
1380-
| "Shuffle Write Block Size Distribution 2" : 0,
1381-
| "Shuffle Write Block Size Distribution 3" : 0,
1382-
| "Shuffle Write Block Size Distribution 4" : 0,
1383-
| "Shuffle Write Block Size Distribution 5" : 0,
1384-
| "Shuffle Write Block Size Distribution 6" : 0,
1385-
| "Shuffle Write Block Size Distribution 7" : 0,
1386-
| "Shuffle Write Block Size Distribution 8" : 0
1336+
| "Shuffle Records Written" : 0
13871337
| },
13881338
| "Input Metrics": {
13891339
| "Bytes Read": 2100,

0 commit comments

Comments
 (0)