8365991: AArch64: Ignore BlockZeroingLowLimit when UseBlockZeroing is false

Signed-off-by: Patrick Zhang <patrick@os.amperecomputing.com>

Author: cnqpzhang

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -6199,23 +6199,22 @@ const int MacroAssembler::zero_words_block_size = 8;
 // ptr, cnt, rscratch1, and rscratch2 are clobbered.
 address MacroAssembler::zero_words(Register ptr, Register cnt)
 {
-  assert(is_power_of_2(zero_words_block_size), "adjust this");
-
   BLOCK_COMMENT("zero_words {");
+  assert(is_power_of_2(zero_words_block_size), "adjust this");
   assert(ptr == r10 && cnt == r11, "mismatch in register usage");
-  RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks());
-  assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated");
 
   subs(rscratch1, cnt, zero_words_block_size);
   Label around;
   br(LO, around);
-  {
+
+  if (UseBlockZeroing) {
+    // Try to zero blocks using DC ZVA if allowed.
     RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::aarch64::zero_blocks());
     assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated");
-    // Make sure this is a C2 compilation. C1 allocates space only for
-    // trampoline stubs generated by Call LIR ops, and in any case it
-    // makes sense for a C1 compilation task to proceed as quickly as
-    // possible.
+    // Ensure that this is a C2 compilation.
+    // The C1 compiler allocates space solely for trampoline stubs generated
+    // by Call LIR operations, and in any case, it is preferable
+    // for a C1 compilation task to complete as quickly as possible.
     CompileTask* task;
     if (StubRoutines::aarch64::complete()
         && Thread::current()->is_Compiler_thread()
@@ -6230,10 +6229,27 @@ address MacroAssembler::zero_words(Register ptr, Register cnt)
       far_call(zero_blocks);
     }
   }
+
+  // Process words with length exceeding the predefined block size threshold.
+  // The loop body will be unrolled based on the number of STP instructions calculated below.
+  const int unroll = zero_words_block_size / 2;
+  Label done, loop;
+  subs(cnt, cnt, unroll * 2);
+  br(Assembler::LT, done);
+  bind(loop);
+  for (int i = 0; i < unroll; i++)
+    stp(zr, zr, post(ptr, 16));
+  subs(cnt, cnt, unroll * 2);
+  br(Assembler::GE, loop);
+  bind(done);
+  add(cnt, cnt, unroll * 2);
+
   bind(around);
 
-  // We have a few words left to do. zero_blocks has adjusted r10 and r11
-  // for us.
+  // A few words remain to complete.
+  // If called, the zero_blocks routine has already performed the necessary
+  // adjustments to registers r10 and r11, ensuring they are correctly set
+  // for subsequent processing.
   for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) {
     Label l;
     tbz(cnt, exact_log2(i), l);
@@ -6259,47 +6275,48 @@ address MacroAssembler::zero_words(Register ptr, Register cnt)
 // r10, r11, rscratch1, and rscratch2 are clobbered.
 address MacroAssembler::zero_words(Register base, uint64_t cnt)
 {
-  assert(wordSize <= BlockZeroingLowLimit,
-            "increase BlockZeroingLowLimit");
+  assert(wordSize <= BlockZeroingLowLimit, "increase BlockZeroingLowLimit");
   address result = nullptr;
-  if (cnt <= (uint64_t)BlockZeroingLowLimit / BytesPerWord) {
+  if (UseBlockZeroing && cnt > (uint64_t)BlockZeroingLowLimit / BytesPerWord) {
+    mov(r10, base); mov(r11, cnt);
+    result = zero_words(r10, r11);
+  } else {
 #ifndef PRODUCT
     {
       char buf[64];
       snprintf(buf, sizeof buf, "zero_words (count = %" PRIu64 ") {", cnt);
       BLOCK_COMMENT(buf);
     }
 #endif
-    if (cnt >= 16) {
-      uint64_t loops = cnt/16;
+    const int block_size = 16;
+    if (cnt >= block_size) {
+      uint64_t loops = cnt/block_size;
       if (loops > 1) {
         mov(rscratch2, loops - 1);
       }
       {
         Label loop;
         bind(loop);
-        for (int i = 0; i < 16; i += 2) {
+        for (int i = 0; i < block_size; i += 2) {
           stp(zr, zr, Address(base, i * BytesPerWord));
         }
-        add(base, base, 16 * BytesPerWord);
+        add(base, base, block_size * BytesPerWord);
         if (loops > 1) {
           subs(rscratch2, rscratch2, 1);
           br(GE, loop);
         }
       }
     }
-    cnt %= 16;
+    cnt %= block_size;
     int i = cnt & 1;  // store any odd word to start
     if (i) str(zr, Address(base));
     for (; i < (int)cnt; i += 2) {
       stp(zr, zr, Address(base, i * wordSize));
     }
     BLOCK_COMMENT("} zero_words");
     result = pc();
-  } else {
-    mov(r10, base); mov(r11, cnt);
-    result = zero_words(r10, r11);
   }
+
   return result;
 }
 

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -642,9 +642,9 @@ class StubGenerator: public StubCodeGenerator {
     return start;
   }
 
-  // The inner part of zero_words().  This is the bulk operation,
-  // zeroing words in blocks, possibly using DC ZVA to do it.  The
-  // caller is responsible for zeroing the last few words.
+  // The inner part of zero_words(). This is the bulk operation,
+  // zeroing words in blocks, using DC ZVA.
+  // The caller is responsible for zeroing the last few words.
   //
   // Inputs:
   // r10: the HeapWord-aligned base address of an array to zero.
@@ -653,10 +653,8 @@ class StubGenerator: public StubCodeGenerator {
   // Returns r10 and r11, adjusted for the caller to clear.
   // r10: the base address of the tail of words left to clear.
   // r11: the number of words in the tail.
-  //      r11 < MacroAssembler::zero_words_block_size.
-
+  //      r11 < MAX2(zva_length * 2, (int)BlockZeroingLowLimit)
   address generate_zero_blocks() {
-    Label done;
     Label base_aligned;
 
     Register base = r10, cnt = r11;
@@ -666,44 +664,27 @@ class StubGenerator: public StubCodeGenerator {
     StubCodeMark mark(this, stub_id);
     address start = __ pc();
 
-    if (UseBlockZeroing) {
-      int zva_length = VM_Version::zva_length();
+    assert (UseBlockZeroing, "only work when UseBlockZeroing is true");
 
-      // Ensure ZVA length can be divided by 16. This is required by
-      // the subsequent operations.
-      assert (zva_length % 16 == 0, "Unexpected ZVA Length");
+    int zva_length = VM_Version::zva_length();
 
-      __ tbz(base, 3, base_aligned);
-      __ str(zr, Address(__ post(base, 8)));
-      __ sub(cnt, cnt, 1);
-      __ bind(base_aligned);
+    // Ensure ZVA length can be divided by 16. This is required by
+    // the subsequent operations.
+    assert (zva_length % 16 == 0, "Unexpected ZVA Length");
 
-      // Ensure count >= zva_length * 2 so that it still deserves a zva after
-      // alignment.
-      Label small;
-      int low_limit = MAX2(zva_length * 2, (int)BlockZeroingLowLimit);
-      __ subs(rscratch1, cnt, low_limit >> 3);
-      __ br(Assembler::LT, small);
-      __ zero_dcache_blocks(base, cnt);
-      __ bind(small);
-    }
+    __ tbz(base, 3, base_aligned);
+    __ str(zr, Address(__ post(base, 8)));
+    __ sub(cnt, cnt, 1);
+    __ bind(base_aligned);
 
-    {
-      // Number of stp instructions we'll unroll
-      const int unroll =
-        MacroAssembler::zero_words_block_size / 2;
-      // Clear the remaining blocks.
-      Label loop;
-      __ subs(cnt, cnt, unroll * 2);
-      __ br(Assembler::LT, done);
-      __ bind(loop);
-      for (int i = 0; i < unroll; i++)
-        __ stp(zr, zr, __ post(base, 16));
-      __ subs(cnt, cnt, unroll * 2);
-      __ br(Assembler::GE, loop);
-      __ bind(done);
-      __ add(cnt, cnt, unroll * 2);
-    }
+    // Ensure count >= zva_length * 2 so that it still deserves a zva after
+    // alignment.
+    Label small;
+    int low_limit = MAX2(zva_length * 2, (int)BlockZeroingLowLimit);
+    __ subs(rscratch1, cnt, low_limit >> 3);
+    __ br(Assembler::LT, small);
+    __ zero_dcache_blocks(base, cnt);
+    __ bind(small);
 
     __ ret(lr);
 

src/hotspot/cpu/aarch64/vm_version_aarch64.cpp

@@ -453,6 +453,9 @@ void VM_Version::initialize() {
     warning("DC ZVA is not available on this CPU");
     FLAG_SET_DEFAULT(UseBlockZeroing, false);
   }
+  if (!UseBlockZeroing && !FLAG_IS_DEFAULT(BlockZeroingLowLimit)) {
+    warning("BlockZeroingLowLimit will not work when UseBlockZeroing is false");
+  }
 
   if (VM_Version::supports_sve2()) {
     if (FLAG_IS_DEFAULT(UseSVE)) {

test/micro/org/openjdk/bench/vm/gc/RawAllocationRate.java

@@ -23,6 +23,8 @@
 package org.openjdk.bench.vm.gc;
 
 import java.util.concurrent.TimeUnit;
+
+import org.openjdk.bench.vm.gc.RawAllocationRate.BaseClass;
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -37,7 +39,7 @@
 @State(Scope.Thread)
 public class RawAllocationRate {
 
-    @Param({"32", "64", "256", "1024", "2048", "4096", "8192", "16384", "65536", "131072"})  // Object size in bytes.
+    @Param({"32", "48", "64", "80", "96", "128", "256", "1024", "2048", "4096", "8192", "16384", "65536", "131072"})  // Object size in bytes.
     public int size;
 
     Object[] objects;
@@ -113,11 +115,16 @@ public Object[] instanceTest_C1() {
 
     static class BaseClass {
     }
-
     static class Class32 extends BaseClass {
        long i0;
        long i1;
     }
+    static class Class48 extends BaseClass {
+       long i0;
+       long i1;
+       long i2;
+       long i3;
+    }
     static class Class64 extends BaseClass {
        long i0;
        long i1;
@@ -126,6 +133,44 @@ static class Class64 extends BaseClass {
        long i4;
        long i5;
     }
+    static class Class80 extends BaseClass {
+       long i0;
+       long i1;
+       long i2;
+       long i3;
+       long i4;
+       long i5;
+       long i6;
+       long i7;
+    }
+    static class Class96 extends BaseClass {
+       long i0;
+       long i1;
+       long i2;
+       long i3;
+       long i4;
+       long i5;
+       long i6;
+       long i7;
+       long i8;
+       long i9;
+    }
+    static class Class128 extends BaseClass {
+       long i0;
+       long i1;
+       long i2;
+       long i3;
+       long i4;
+       long i5;
+       long i6;
+       long i7;
+       long i8;
+       long i9;
+       long i10;
+       long i11;
+       long i12;
+       long i13;
+    }
     static class Class256 extends BaseClass {
        long i0;
        long i1;
@@ -28707,8 +28752,16 @@ static final BaseClass newInstance(int size) {
         switch (size) {
         case 32:
             return new Class32();
+        case 48:
+            return new Class48();
         case 64:
             return new Class64();
+        case 80:
+            return new Class80();
+        case 96:
+            return new Class96();
+        case 128:
+            return new Class128();
         case 256:
             return new Class256();
         case 1024: