[SPARK-30604 Add HostLocalBlock size in log total bytes]

Udbhav30 · Udbhav30 · commit 5331053b3cdd · 2020-01-22T21:18:57.000+05:30
diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -317,7 +317,7 @@ final class ShuffleBlockFetcherIterator(
         collectFetchRequests(address, blockInfos, collectedRemoteRequests)
       }
     }
-    val totalBytes = localBlockBytes + remoteBlockBytes
+    val totalBytes = localBlockBytes + remoteBlockBytes + hostLocalBlockBytes
     logInfo(s"Getting $numBlocksToFetch (${Utils.bytesToString(totalBytes)}) non-empty blocks " +
       s"including ${localBlocks.size} (${Utils.bytesToString(localBlockBytes)}) local and " +
       s"${hostLocalBlocks.size} (${Utils.bytesToString(hostLocalBlockBytes)}) " +

Original file line number	Diff line number	Diff line change
`@@ -317,7 +317,7 @@ final class ShuffleBlockFetcherIterator(`
`317`	`317`	`collectFetchRequests(address, blockInfos, collectedRemoteRequests)`
`318`	`318`	`}`
`319`	`319`	`}`
`320`		`- val totalBytes = localBlockBytes + remoteBlockBytes`
	`320`	`+ val totalBytes = localBlockBytes + remoteBlockBytes + hostLocalBlockBytes`
`321`	`321`	`logInfo(s"Getting $numBlocksToFetch (${Utils.bytesToString(totalBytes)}) non-empty blocks " +`
`322`	`322`	`s"including ${localBlocks.size} (${Utils.bytesToString(localBlockBytes)}) local and " +`
`323`	`323`	`s"${hostLocalBlocks.size} (${Utils.bytesToString(hostLocalBlockBytes)}) " +`