Add Xgc options for suballocator heap size and quick allocation

1. Use VMEM_ALLOC_QUICK by default for allocateRegion in
   allocate_memory32
2. Adds -Xgc:suballocatorQuickAllocDisable option that
   disables the default VMEM_ALLOC_QUICK
3. Adds -Xgc:suballocatorIncrementSize option that replaces the
   HEAP_SIZE_BYTES macro and controls the heap increment size

Addresses: #7190
Signed-off-by: Nathan Henderson <nathan.henderson@ibm.com>

fvtest/porttest/omrmemTest.cpp

@@ -56,8 +56,8 @@ extern PortTestEnvironment *portTestEnv;
 #endif
 
 #if defined(OMR_ENV_DATA64)
-/* this macro corresponds to the one defined in omrmem32helpers */
-#define HEAP_SIZE_BYTES 8*1024*1024
+/* This macro corresponds to SUBALLOCATOR_INCREMENT_SIZE defined in omrgcconsts.h. */
+#define HEAP_SIZE_BYTES (8 * 1024 * 1024)
 #endif
 
 #define COMPLETE_LARGE_REGION 1

gc/base/GCExtensionsBase.hpp

@@ -743,8 +743,10 @@ class MM_GCExtensionsBase : public MM_BaseVirtual {
 	};
 	HeapInitializationFailureReason heapInitializationFailureReason; /**< Error code provided additional information about heap initialization failure */
 	bool scavengerAlignHotFields; /**< True if the scavenger is to check the hot field description for an object in order to better cache align it when tenuring (enabled with the -Xgc:hotAlignment option) */
-	uintptr_t suballocatorInitialSize; /**< the initial chunk size in bytes for the J9Heap suballocator (enabled with the -Xgc:suballocatorInitialSize option) */
-	uintptr_t suballocatorCommitSize; /**< the commit size in bytes for the J9Heap suballocator (enabled with the -Xgc:suballocatorCommitSize option) */
+	uintptr_t suballocatorInitialSize; /**< the initial chunk size in bytes for the heap suballocator (enabled with the -Xgc:suballocatorInitialSize option) */
+	uintptr_t suballocatorCommitSize; /**< the commit size in bytes for the heap suballocator (enabled with the -Xgc:suballocatorCommitSize option) */
+	uintptr_t suballocatorIncrementSize; /**< the increment size in bytes for the heap suballocator (enabled with the -Xgc:suballocatorIncrementSize option) */
+	bool suballocatorQuickAlloc; /**< use OMRPORT_VMEM_ALLOC_QUICK for the heap suballocator (disabled with the -Xgc:suballocatorQuickAllocDisable option) (Linux only) */
 
 #if defined(OMR_GC_COMPRESSED_POINTERS)
 	bool shouldAllowShiftingCompression; /**< temporary option to enable compressed reference scaling by shifting pointers */
@@ -1868,8 +1870,10 @@ class MM_GCExtensionsBase : public MM_BaseVirtual {
 		, heapCeiling(0) /* default for normal platforms is 0 (i.e. no ceiling) */
 		, heapInitializationFailureReason(HEAP_INITIALIZATION_FAILURE_REASON_NO_ERROR)
 		, scavengerAlignHotFields(true) /* VM Design 1774: hot field alignment is on by default */
-		, suballocatorInitialSize(SUBALLOCATOR_INITIAL_SIZE) /* default for J9Heap suballocator initial size is 200 MB */
-		, suballocatorCommitSize(SUBALLOCATOR_COMMIT_SIZE) /* default for J9Heap suballocator commit size is 50 MB */
+		, suballocatorInitialSize(SUBALLOCATOR_INITIAL_SIZE) /* default for heap suballocator initial size is 200 MB */
+		, suballocatorCommitSize(SUBALLOCATOR_COMMIT_SIZE) /* default for heap suballocator commit size is 50 MB */
+		, suballocatorIncrementSize(SUBALLOCATOR_INCREMENT_SIZE) /* default for heap suballocator commit size is 8 MB or 256 MB for AIX */
+		, suballocatorQuickAlloc(true) /* use mmap-based allocation by default for the heap suballocator (Linux only) */
 #if defined(OMR_GC_COMPRESSED_POINTERS)
 		, shouldAllowShiftingCompression(true) /* VM Design 1810: shifting compression enabled, by default, for compressed refs */
 		, shouldForceSpecifiedShiftingCompression(0)

gc/base/MemoryManager.cpp

@@ -171,9 +171,15 @@ MM_MemoryManager::createVirtualMemoryForHeap(MM_EnvironmentBase *env, MM_MemoryH
 			bool shouldHeapBeAllocatedFirst = (NULL != preferredAddress);
 			void *startAllocationAddress = preferredAddress;
 
-			/* Set the commit size for the sub allocator. This needs to be completed before the call to omrmem_ensure_capacity32 */
+			/* Set the commit size for the suballocator. This needs to be completed before the call to omrmem_ensure_capacity32. */
 			omrport_control(OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE, extensions->suballocatorCommitSize);
 
+			/* Set the increment size for the suballocator. This needs to be completed before the call to omrmem_ensure_capacity32. */
+			omrport_control(OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE, extensions->suballocatorIncrementSize);
+
+			/* Set if the suballocator should use ALLOC_QUICK. This needs to be completed before the call to omrmem_ensure_capacity32. */
+			omrport_control(OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC, extensions->suballocatorQuickAlloc ? 1 : 0);
+
 			if (!shouldHeapBeAllocatedFirst) {
 				if (OMRPORT_ENSURE_CAPACITY_FAILED == omrmem_ensure_capacity32(extensions->suballocatorInitialSize)) {
 					extensions->heapInitializationFailureReason = MM_GCExtensionsBase::HEAP_INITIALIZATION_FAILURE_REASON_CAN_NOT_ALLOCATE_LOW_MEMORY_RESERVE;

include_core/omrgcconsts.h

@@ -555,15 +555,28 @@ typedef enum {
 #define PREFERRED_HEAP_BASE 0x0
 #endif
 
-#define SUBALLOCATOR_INITIAL_SIZE (200*1024*1024)
-#define SUBALLOCATOR_COMMIT_SIZE (50*1024*1024)
+#define SUBALLOCATOR_INITIAL_SIZE (200 * 1024 * 1024)
+#define SUBALLOCATOR_COMMIT_SIZE (50 * 1024 * 1024)
 #if defined(AIXPPC)
 /* virtual memory is assigned in segment of 256M, so grab the entire segment */
-#define SUBALLOCATOR_ALIGNMENT (256*1024*1024)
+#define SUBALLOCATOR_ALIGNMENT (256 * 1024 * 1024)
 #else /* defined(AIXPPC) */
-#define SUBALLOCATOR_ALIGNMENT (8*1024*1024)
+#define SUBALLOCATOR_ALIGNMENT (8 * 1024 * 1024)
 #endif /* defined(AIXPPC) */
 
+/* VMDESIGN 1761 The size of a suballocation heap.
+ * See VMDESIGN 1761 for the rationale behind the selection of this size.
+ * An 8MB heap is used to provide more capacity in the cases where an application loads a greater
+ * number of classes than typical.
+ * For testing purposes, this value is mirrored in the port library test. omrmemTest.cpp should be
+ * updated if this value is changed.
+ */
+#if defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS)
+#define SUBALLOCATOR_INCREMENT_SIZE (256 * 1024 * 1024)
+#else /* defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS) */
+#define SUBALLOCATOR_INCREMENT_SIZE (8 * 1024 * 1024)
+#endif /* defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS) */
+
 #define MAXIMUM_HEAP_SIZE_RECOMMENDED_FOR_COMPRESSEDREFS            ((U_64)57 * 1024 * 1024 * 1024)
 #define MAXIMUM_HEAP_SIZE_RECOMMENDED_FOR_3BIT_SHIFT_COMPRESSEDREFS ((U_64)25 * 1024 * 1024 * 1024)
 

include_core/omrport.h

@@ -877,6 +877,8 @@ typedef struct J9ProcessorInfos {
 #define OMRPORT_CTLDATA_MEM_CATEGORIES_SET  "MEM_CATEGORIES_SET"
 #define OMRPORT_CTLDATA_AIX_PROC_ATTR  "AIX_PROC_ATTR"
 #define OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE  "ALLOCATE32_COMMIT_SIZE"
+#define OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE  "ALLOCATE32_INCREMENT_SIZE"
+#define OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC  "ALLOCATE32_QUICK_ALLOC"
 #define OMRPORT_CTLDATA_NOSUBALLOC32BITMEM  "NOSUBALLOC32BITMEM"
 #define OMRPORT_CTLDATA_VMEM_ADVISE_OS_ONFREE  "VMEM_ADVISE_OS_ONFREE"
 #define OMRPORT_CTLDATA_VECTOR_REGS_SUPPORT_ON  "VECTOR_REGS_SUPPORT_ON"

port/common/omrmem32helpers.c

@@ -21,6 +21,7 @@
  *******************************************************************************/
 
 #include "omrmem32helpers.h"
+#include "omrgcconsts.h"
 #include "omrport.h"
 #include "omrportpg.h"
 #include "ut_omrport.h"
@@ -37,26 +38,14 @@ static void *reserveAndCommitRegion(struct OMRPortLibrary *portLibrary, uintptr_
 #define VMEM_MODE_COMMIT OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE | OMRPORT_VMEM_MEMORY_MODE_COMMIT
 #define VMEM_MODE_WITHOUT_COMMIT OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE
 
+#define MEM32_LIMIT ((uintptr_t)0XFFFFFFFFU)
 struct {
 	uintptr_t base;
 	uintptr_t limit;
 } regions[] = {
-	{0x0, 0xFFFFFFFF}
+	{0x0, MEM32_LIMIT}
 };
 
-#define MEM32_LIMIT 0XFFFFFFFF
-
-/* VMDESIGN 1761 The size of a suballocation heap.
- * See VMDESIGN 1761 for the rationale behind the selection of this size.
- * We use a 8MB heap to give us more room in case an application loads a larger amount of classes than usual.
- * For testing purposes, this value is mirrored in port library test. If we tune this value, we should also adjust it in omrmemTest.cpp
- */
-#if defined(AIXPPC) && defined(OMR_GC_COMPRESSED_POINTERS)
-/* virtual memory is allocated in 256M segments on AIX, so grab the whole segment */
-#define HEAP_SIZE_BYTES (256 * 1024 * 1024)
-#else
-#define HEAP_SIZE_BYTES (8 * 1024 * 1024)
-#endif
 /* Creates any of the resources required to use allocate_memory32
  *
  * Note: Any resources created here need to be cleaned up in shutdown_memory32_using_vmem
@@ -71,7 +60,9 @@ startup_memory32(struct OMRPortLibrary *portLibrary)
 	PPG_mem_mem32_subAllocHeapMem32.subCommitCommittedMemorySize = 0;
 	PPG_mem_mem32_subAllocHeapMem32.subCommitHeapWrapper = NULL;
 	PPG_mem_mem32_subAllocHeapMem32.suballocator_initialSize = 0;
-	PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize = 0;
+	PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize = SUBALLOCATOR_COMMIT_SIZE;
+	PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize = SUBALLOCATOR_INCREMENT_SIZE;
+	PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc = TRUE;
 
 	/* initialize the monitor in subAllocHeap32 */
 	if (0 != omrthread_monitor_init(&(PPG_mem_mem32_subAllocHeapMem32.monitor), 0)) {
@@ -438,15 +429,26 @@ allocate_memory32(struct OMRPortLibrary *portLibrary, uintptr_t byteAmount, cons
 #endif
 		omrthread_monitor_enter(PPG_mem_mem32_subAllocHeapMem32.monitor);
 
-		/* Check if byteAmount is larger than HEAP_SIZE_BYTES.
+		/* Check if byteAmount is larger than PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize.
 		 * The majority of size requests will typically be much smaller.
 		 */
 		returnPtr = iterateHeapsAndSubAllocate(portLibrary, byteAmount);
 		if (NULL == returnPtr) {
-			if (byteAmount >= HEAP_SIZE_BYTES) {
-				returnPtr = allocateLargeRegion(portLibrary, byteAmount, callSite, 0);
+			if (byteAmount >= PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize) {
+				returnPtr = allocateLargeRegion(
+							portLibrary,
+							byteAmount,
+							callSite,
+							0);
 			} else {
-				returnPtr = allocateRegion(portLibrary, HEAP_SIZE_BYTES, byteAmount, callSite, 0);
+				/* For 64-bit Linux, use the OMRPORT_VMEM_ALLOC_QUICK flag if it has not been disabled. */
+				uintptr_t vmemAllocOptions = PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc ? OMRPORT_VMEM_ALLOC_QUICK : 0;
+				returnPtr = allocateRegion(
+							portLibrary,
+							PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize,
+							byteAmount,
+							callSite,
+							vmemAllocOptions);
 			}
 		}
 
@@ -466,11 +468,11 @@ ensure_capacity32(struct OMRPortLibrary *portLibrary, uintptr_t byteAmount)
 	J9HeapWrapper *heapWrapperCursor = NULL;
 	uintptr_t returnValue = OMRPORT_ENSURE_CAPACITY_FAILED;
 #if defined(OMR_ENV_DATA64)
-	/* For 64 bit os, use flag OMRPORT_VMEM_ALLOC_QUICK as it is in the startup period. */
+	/* For 64-bit OS, use the OMRPORT_VMEM_ALLOC_QUICK flag during startup. */
 	uintptr_t vmemAllocOptions = OMRPORT_VMEM_ALLOC_QUICK;
-#else
+#else /* defined(OMR_ENV_DATA64) */
 	uintptr_t vmemAllocOptions = 0;
-#endif
+#endif /* defined(OMR_ENV_DATA64) */
 
 	Trc_PRT_mem_ensure_capacity32_Entry(byteAmount);
 
@@ -481,9 +483,9 @@ ensure_capacity32(struct OMRPortLibrary *portLibrary, uintptr_t byteAmount)
 	}
 #endif
 
-	/* Ensured byte amount should be at least HEAP_SIZE_BYTES large */
-	if (byteAmount < HEAP_SIZE_BYTES) {
-		byteAmount = HEAP_SIZE_BYTES;
+	/* Ensured byte amount should be at least PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize large. */
+	if (byteAmount < PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize) {
+		byteAmount = PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize;
 	}
 
 	omrthread_monitor_enter(PPG_mem_mem32_subAllocHeapMem32.monitor);

port/common/omrmem32struct.h

@@ -41,6 +41,8 @@ typedef struct J9SubAllocateHeapMem32 {
 	J9HeapWrapper *subCommitHeapWrapper;
 	uintptr_t suballocator_initialSize;
 	uintptr_t suballocator_commitSize;
+	uintptr_t suballocator_incrementSize;
+	BOOLEAN suballocator_quickAlloc;
 } J9SubAllocateHeapMem32;
 
 #endif	/* omrmem32struct_h */

port/common/omrportcontrol.c

@@ -54,26 +54,37 @@ omrport_control(struct OMRPortLibrary *portLibrary, const char *key, uintptr_t v
 		return 0;
 	}
 #if defined(OMR_ENV_DATA64)
-	if (!strcmp(OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE, key)) {
+	if (0 == strcmp(OMRPORT_CTLDATA_ALLOCATE32_COMMIT_SIZE, key)) {
 		if (0 != value) {
-			/* CommitSize is immutable. It can only be set once. */
-			if (0 == PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize) {
-				/* Round up the commit size to the page size and set it to global variable */
-				uintptr_t pageSize = portLibrary->vmem_supported_page_sizes(portLibrary)[0];
-				uintptr_t roundedCommitSize = pageSize * (value / pageSize);
-				if (roundedCommitSize < value) {
-					roundedCommitSize += pageSize;
-				}
-				PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize = roundedCommitSize;
-			} else {
-				return 1;
+			/* Round up the commit size to the page size and set it to global variable. */
+			uintptr_t pageSize = portLibrary->vmem_supported_page_sizes(portLibrary)[0];
+			uintptr_t roundedCommitSize = pageSize * (value / pageSize);
+			if (roundedCommitSize < value) {
+				roundedCommitSize += pageSize;
 			}
+			PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize = roundedCommitSize;
 		} else {
 			return (int32_t)PPG_mem_mem32_subAllocHeapMem32.suballocator_commitSize;
 		}
 		return 0;
+	} else if (0 == strcmp(OMRPORT_CTLDATA_ALLOCATE32_INCREMENT_SIZE, key)) {
+		if (0 != value) {
+			/* Round up the increment size to the page size and set it to global variable. */
+			uintptr_t pageSize = portLibrary->vmem_supported_page_sizes(portLibrary)[0];
+			uintptr_t roundedIncrementSize = pageSize * (value / pageSize);
+			if (roundedIncrementSize < value) {
+				roundedIncrementSize += pageSize;
+			}
+			PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize = roundedIncrementSize;
+		} else {
+			return (int32_t)PPG_mem_mem32_subAllocHeapMem32.suballocator_incrementSize;
+		}
+		return 0;
+	} else if (0 == strcmp(OMRPORT_CTLDATA_ALLOCATE32_QUICK_ALLOC, key)) {
+		PPG_mem_mem32_subAllocHeapMem32.suballocator_quickAlloc = (0 != value) ? TRUE : FALSE;
+		return 0;
 	}
-#endif
+#endif /* defined(OMR_ENV_DATA64) */
 
 #if defined(OMR_RAS_TDF_TRACE)
 	if (!strcmp(OMRPORT_CTLDATA_TRACE_START, key) && value) {