Skip to content

Commit c03db87

Browse files
committed
Add a test case to check memory resource include offheap part
1 parent 66ae9a1 commit c03db87

File tree

1 file changed

+21
-0
lines changed

1 file changed

+21
-0
lines changed

resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -514,4 +514,25 @@ class YarnAllocatorSuite extends SparkFunSuite with Matchers with BeforeAndAfter
514514
verify(rmClientSpy)
515515
.updateBlacklist(hosts.slice(10, 11).asJava, Collections.emptyList())
516516
}
517+
518+
test("SPARK-28577#YarnAllocator.resource.memory should include offHeapSize " +
519+
"when offHeapEnabled is true.") {
520+
val originalOffHeapEnabled = sparkConf.get(MEMORY_OFFHEAP_ENABLED)
521+
val originalOffHeapSize = sparkConf.get(MEMORY_OFFHEAP_SIZE)
522+
val executorMemory = sparkConf.get(EXECUTOR_MEMORY).toInt
523+
val offHeapMemoryInMB = 1024L
524+
val offHeapMemoryInByte = offHeapMemoryInMB * 1024 * 1024
525+
try {
526+
sparkConf.set(MEMORY_OFFHEAP_ENABLED, true)
527+
sparkConf.set(MEMORY_OFFHEAP_SIZE, offHeapMemoryInByte)
528+
val allocator = createAllocator(maxExecutors = 1,
529+
additionalConfigs = Map(EXECUTOR_MEMORY.key -> executorMemory.toString))
530+
val memory = allocator.resource.getMemory
531+
assert(memory ==
532+
executorMemory + offHeapMemoryInMB + YarnSparkHadoopUtil.MEMORY_OVERHEAD_MIN)
533+
} finally {
534+
sparkConf.set(MEMORY_OFFHEAP_ENABLED, originalOffHeapEnabled)
535+
sparkConf.set(MEMORY_OFFHEAP_SIZE, originalOffHeapSize)
536+
}
537+
}
517538
}

0 commit comments

Comments
 (0)