[GC] 8252500: ZGC on aarch64: Unable to allocate heap for certain Lin…

…ux kernel configurations

Summary: Backport JDK-8252500 for ZGC on aarch64.
    openjdk/jdk@73ba3ae

Reviewed-by: albert.th

Test Plan: test/hotspot/jtreg/gc/z/

Issue: #173

src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp

@@ -25,8 +25,13 @@
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zUtils.inline.hpp"
 #include "runtime/globals.hpp"
+#include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
 
+#ifdef LINUX
+#include <sys/mman.h>
+#endif // LINUX
+
 //
 // The heap can have three different layouts, depending on the max heap size.
 //
@@ -135,6 +140,92 @@
 //  * 63-48 Fixed (16-bits, always zero)
 //
 
+// Default value if probing is not implemented for a certain platform: 128TB
+static const size_t DEFAULT_MAX_ADDRESS_BIT = 47;
+// Minimum value returned, if probing fails: 64GB
+static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
+
+size_t count_leading_zeros(size_t addr) {
+  size_t n = 1;
+  if (addr > 0xffffffff) {
+    addr >>= 32;
+    n += 32;
+  }
+  if (addr > 0xffff) {
+    addr >>= 16;
+    n += 16;
+  }
+  if (addr > 0xff) {
+    addr >>= 8;
+    n += 8;
+  }
+  if (addr > 0xf) {
+    addr >>= 4;
+    n += 4;
+  }
+  if (addr > 0x3) {
+    addr >>= 2;
+    n += 2;
+  }
+  if (addr > 0x1) {
+    addr >>= 1;
+    n += 1;
+  }
+  if (addr == 0) {
+    n = 0;
+  }
+  return 64 - n;
+}
+
+static size_t probe_valid_max_address_bit() {
+#ifdef LINUX
+  size_t max_address_bit = 0;
+  const size_t page_size = os::vm_page_size();
+  for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
+    const uintptr_t base_addr = ((uintptr_t) 1U) << i;
+    if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
+      // msync suceeded, the address is valid, and maybe even already mapped.
+      max_address_bit = i;
+      break;
+    }
+    if (errno != ENOMEM) {
+      // Some error occured. This should never happen, but msync
+      // has some undefined behavior, hence ignore this bit.
+#ifdef ASSERT
+      fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#else // ASSERT
+      log_warning(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#endif // ASSERT
+      continue;
+    }
+    // Since msync failed with ENOMEM, the page might not be mapped.
+    // Try to map it, to see if the address is valid.
+    void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      munmap(result_addr, page_size);
+    }
+    if ((uintptr_t) result_addr == base_addr) {
+      // address is valid
+      max_address_bit = i;
+      break;
+    }
+  }
+  if (max_address_bit == 0) {
+    // probing failed, allocate a very high page and take that bit as the maximum
+    const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
+    void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
+      munmap(result_addr, page_size);
+    }
+  }
+  log_info(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
+  return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
+#else // LINUX
+  return DEFAULT_MAX_ADDRESS_BIT;
+#endif // LINUX
+}
+
 uintptr_t ZPlatformAddressSpaceStart() {
   const uintptr_t first_heap_view_address = (uintptr_t)1 << (ZPlatformAddressMetadataShift() + 0);
   const size_t min_address_offset = 0;
@@ -156,8 +247,9 @@ uintptr_t ZPlatformAddressReservedEnd() {
 }
 
 size_t ZPlatformAddressOffsetBits() {
-  const size_t min_address_offset_bits = 42; // 4TB
-  const size_t max_address_offset_bits = 44; // 16TB
+  const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
+  const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
+  const size_t min_address_offset_bits = max_address_offset_bits - 2;
   const size_t virtual_to_physical_ratio = 7; // 7:1
   const size_t address_offset = ZUtils::round_up_power_of_2(MaxHeapSize * virtual_to_physical_ratio);
   const size_t address_offset_bits = log2_intptr(address_offset);

src/hotspot/share/gc/z/zVirtualMemory.cpp

@@ -31,8 +31,13 @@ ZVirtualMemoryManager::ZVirtualMemoryManager() :
     _manager(),
     _initialized(false) {
 
-  log_info(gc, init)("Address Space: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "T)",
-                     ZAddressSpaceStart, ZAddressSpaceEnd, ZAddressSpaceSize / K / G);
+  if (ZAddressSpaceSize / K / G != 0) {
+    log_info(gc, init)("Address Space: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "T)",
+                       ZAddressSpaceStart, ZAddressSpaceEnd, ZAddressSpaceSize / K / G);
+  } else {
+    log_info(gc, init)("Address Space: " PTR_FORMAT " - " PTR_FORMAT " (" SIZE_FORMAT "G)",
+                       ZAddressSpaceStart, ZAddressSpaceEnd, ZAddressSpaceSize / G);
+  }
 
   // Reserve address space
   if (!reserve(ZAddressSpaceStart, ZAddressSpaceSize)) {