Skip to content

Commit

Permalink
[SPARK-25299] fix reader benchmarks (#544)
Browse files Browse the repository at this point in the history
Fix the stubbing of the reader benchmark tests
  • Loading branch information
yifeih authored and bulldozer-bot[bot] committed May 1, 2019
1 parent b35d238 commit 3a760e7
Showing 1 changed file with 13 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,12 @@ import org.mockito.{Mock, MockitoAnnotations}
import org.mockito.Answers.RETURNS_SMART_NULLS
import org.mockito.ArgumentMatchers.any
import org.mockito.Mockito.when
import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer
import scala.util.Random

import org.apache.spark.{Aggregator, MapOutputTracker, ShuffleDependency, SparkConf, SparkEnv, TaskContext}
import org.apache.spark.api.shuffle.ShuffleLocation
import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
import org.apache.spark.executor.TaskMetrics
import org.apache.spark.memory.{TaskMemoryManager, TestMemoryManager}
Expand Down Expand Up @@ -194,14 +197,17 @@ object BlockStoreShuffleReaderBenchmark extends BenchmarkBase {
}

when(mapOutputTracker.getMapSizesByShuffleLocation(0, 0, 1))
.thenReturn {
val shuffleBlockIdsAndSizes = (0 until NUM_MAPS).map { mapId =>
val shuffleBlockId = ShuffleBlockId(0, mapId, 0)
(shuffleBlockId, dataFileLength)
.thenAnswer(new Answer[Iterator[(Option[ShuffleLocation], Seq[(BlockId, Long)])]] {
def answer(invocationOnMock: InvocationOnMock):
Iterator[(Option[ShuffleLocation], Seq[(BlockId, Long)])] = {
val shuffleBlockIdsAndSizes = (0 until NUM_MAPS).map { mapId =>
val shuffleBlockId = ShuffleBlockId(0, mapId, 0)
(shuffleBlockId, dataFileLength)
}
Seq((Option.apply(DefaultMapShuffleLocations.get(dataBlockId)), shuffleBlockIdsAndSizes))
.toIterator
}
Seq((Option.apply(DefaultMapShuffleLocations.get(dataBlockId)), shuffleBlockIdsAndSizes))
.toIterator
}
})

when(dependency.serializer).thenReturn(serializer)
when(dependency.aggregator).thenReturn(aggregator)
Expand Down

0 comments on commit 3a760e7

Please sign in to comment.