File tree Expand file tree Collapse file tree 1 file changed +21
-0
lines changed
resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn Expand file tree Collapse file tree 1 file changed +21
-0
lines changed Original file line number Diff line number Diff line change @@ -514,4 +514,25 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
514514 verify(rmClientSpy)
515515 .updateBlacklist(hosts.slice(10 , 11 ).asJava, Collections .emptyList())
516516 }
517+
518+ test(" SPARK-28577#YarnAllocator.resource.memory should include offHeapSize " +
519+ " when offHeapEnabled is true." ) {
520+ val originalOffHeapEnabled = sparkConf.get(MEMORY_OFFHEAP_ENABLED )
521+ val originalOffHeapSize = sparkConf.get(MEMORY_OFFHEAP_SIZE )
522+ val executorMemory = sparkConf.get(EXECUTOR_MEMORY ).toInt
523+ val offHeapMemoryInMB = 1024L
524+ val offHeapMemoryInByte = offHeapMemoryInMB * 1024 * 1024
525+ try {
526+ sparkConf.set(MEMORY_OFFHEAP_ENABLED , true )
527+ sparkConf.set(MEMORY_OFFHEAP_SIZE , offHeapMemoryInByte)
528+ val allocator = createAllocator(maxExecutors = 1 ,
529+ additionalConfigs = Map (EXECUTOR_MEMORY .key -> executorMemory.toString))
530+ val memory = allocator.resource.getMemory
531+ assert(memory ==
532+ executorMemory + offHeapMemoryInMB + YarnSparkHadoopUtil .MEMORY_OVERHEAD_MIN )
533+ } finally {
534+ sparkConf.set(MEMORY_OFFHEAP_ENABLED , originalOffHeapEnabled)
535+ sparkConf.set(MEMORY_OFFHEAP_SIZE , originalOffHeapSize)
536+ }
537+ }
517538}
You can’t perform that action at this time.
0 commit comments