8343840: Rewrite the ObjectMonitor lists

Reviewed-by: dholmes, coleenp, pchilanomate, yzheng

Author: fbredber

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -312,10 +312,8 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
   // StoreLoad achieves this.
   membar(StoreLoad);
 
-  // Check if the entry lists are empty (EntryList first - by convention).
-  ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
-  ldr(tmpReg, Address(tmp, ObjectMonitor::cxq_offset()));
-  orr(rscratch1, rscratch1, tmpReg);
+  // Check if the entry_list is empty.
+  ldr(rscratch1, Address(tmp, ObjectMonitor::entry_list_offset()));
   cmp(rscratch1, zr);
   br(Assembler::EQ, cont);     // If so we are done.
 
@@ -635,10 +633,8 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi
     // StoreLoad achieves this.
     membar(StoreLoad);
 
-    // Check if the entry lists are empty (EntryList first - by convention).
-    ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
-    ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
-    orr(rscratch1, rscratch1, t3_t);
+    // Check if the entry_list is empty.
+    ldr(rscratch1, Address(t1_monitor, ObjectMonitor::entry_list_offset()));
     cmp(rscratch1, zr);
     br(Assembler::EQ, unlocked);  // If so we are done.
 

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -2958,10 +2958,8 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
   // StoreLoad achieves this.
   membar(StoreLoad);
 
-  // Check if the entry lists are empty (EntryList first - by convention).
-  ld(temp,             in_bytes(ObjectMonitor::EntryList_offset()), current_header);
-  ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
-  orr(temp, temp, displaced_header); // Will be 0 if both are 0.
+  // Check if the entry_list is empty.
+  ld(temp, in_bytes(ObjectMonitor::entry_list_offset()), current_header);
   cmpdi(flag, temp, 0);
   beq(flag, success);  // If so we are done.
 
@@ -3299,8 +3297,6 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister f
 
     bind(not_recursive);
 
-    const Register t2 = tmp2;
-
     // Set owner to null.
     // Release to satisfy the JMM
     release();
@@ -3310,10 +3306,8 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister f
     // StoreLoad achieves this.
     membar(StoreLoad);
 
-    // Check if the entry lists are empty (EntryList first - by convention).
-    ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
-    ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
-    orr(t, t, t2);
+    // Check if the entry_list is empty.
+    ld(t, in_bytes(ObjectMonitor::entry_list_offset()), monitor);
     cmpdi(CR0, t, 0);
     beq(CR0, unlocked); // If so we are done.
 

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp

@@ -233,10 +233,8 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg,
   // StoreLoad achieves this.
   membar(StoreLoad);
 
-  // Check if the entry lists are empty (EntryList first - by convention).
-  ld(t0, Address(tmp, ObjectMonitor::EntryList_offset()));
-  ld(tmp1Reg, Address(tmp, ObjectMonitor::cxq_offset()));
-  orr(t0, t0, tmp1Reg);
+  // Check if the entry_list is empty.
+  ld(t0, Address(tmp, ObjectMonitor::entry_list_offset()));
   beqz(t0, unlocked); // If so we are done.
 
   // Check if there is a successor.
@@ -569,10 +567,8 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box,
     // StoreLoad achieves this.
     membar(StoreLoad);
 
-    // Check if the entry lists are empty (EntryList first - by convention).
-    ld(t0, Address(tmp1_monitor, ObjectMonitor::EntryList_offset()));
-    ld(tmp3_t, Address(tmp1_monitor, ObjectMonitor::cxq_offset()));
-    orr(t0, t0, tmp3_t);
+    // Check if the entry_list is empty.
+    ld(t0, Address(tmp1_monitor, ObjectMonitor::entry_list_offset()));
     beqz(t0, unlocked); // If so we are done.
 
     // Check if there is a successor.

src/hotspot/cpu/s390/macroAssembler_s390.cpp

@@ -3931,7 +3931,7 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg
 
   bind(not_recursive);
 
-  NearLabel check_succ, set_eq_unlocked;
+  NearLabel set_eq_unlocked;
 
   // Set owner to null.
   // Release to satisfy the JMM
@@ -3941,14 +3941,10 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg
   // We need a full fence after clearing owner to avoid stranding.
   z_fence();
 
-  // Check if the entry lists are empty (EntryList first - by convention).
-  load_and_test_long(temp, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
-  z_brne(check_succ);
-  load_and_test_long(temp, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
+  // Check if the entry_list is empty.
+  load_and_test_long(temp, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(entry_list)));
   z_bre(done); // If so we are done.
 
-  bind(check_succ);
-
   // Check if there is a successor.
   load_and_test_long(temp, Address(currentHeader, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)));
   z_brne(set_eq_unlocked); // If so we are done.
@@ -6794,9 +6790,8 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(Register obj, Regis
 
     const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
     const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
-    const Address cxq_address{monitor, ObjectMonitor::cxq_offset() - monitor_tag};
     const Address succ_address{monitor, ObjectMonitor::succ_offset() - monitor_tag};
-    const Address EntryList_address{monitor, ObjectMonitor::EntryList_offset() - monitor_tag};
+    const Address entry_list_address{monitor, ObjectMonitor::entry_list_offset() - monitor_tag};
     const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
 
     NearLabel not_recursive;
@@ -6813,7 +6808,7 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(Register obj, Regis
 
     bind(not_recursive);
 
-    NearLabel check_succ, set_eq_unlocked;
+    NearLabel set_eq_unlocked;
 
     // Set owner to null.
     // Release to satisfy the JMM
@@ -6823,14 +6818,10 @@ void MacroAssembler::compiler_fast_unlock_lightweight_object(Register obj, Regis
     // We need a full fence after clearing owner to avoid stranding.
     z_fence();
 
-    // Check if the entry lists are empty (EntryList first - by convention).
-    load_and_test_long(tmp2, EntryList_address);
-    z_brne(check_succ);
-    load_and_test_long(tmp2, cxq_address);
+    // Check if the entry_list is empty.
+    load_and_test_long(tmp2, entry_list_address);
     z_bre(unlocked); // If so we are done.
 
-    bind(check_succ);
-
     // Check if there is a successor.
     load_and_test_long(tmp2, succ_address);
     z_brne(set_eq_unlocked); // If so we are done.

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -414,8 +414,6 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
   // Despite our balanced locking property we still check that m->_owner == Self
   // as java routines or native JNI code called by this thread might
   // have released the lock.
-  // Refer to the comments in synchronizer.cpp for how we might encode extra
-  // state in _succ so we can avoid fetching EntryList|cxq.
   //
   // If there's no contention try a 1-0 exit.  That is, exit without
   // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
@@ -447,9 +445,8 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
   // StoreLoad achieves this.
   membar(StoreLoad);
 
-  // Check if the entry lists are empty (EntryList first - by convention).
-  movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
-  orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
+  // Check if the entry_list is empty.
+  cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(entry_list)), NULL_WORD);
   jccb(Assembler::zero, LSuccess);    // If so we are done.
 
   // Check if there is a successor.
@@ -767,9 +764,8 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
     }
     const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
     const Address recursions_address{monitor, ObjectMonitor::recursions_offset() - monitor_tag};
-    const Address cxq_address{monitor, ObjectMonitor::cxq_offset() - monitor_tag};
     const Address succ_address{monitor, ObjectMonitor::succ_offset() - monitor_tag};
-    const Address EntryList_address{monitor, ObjectMonitor::EntryList_offset() - monitor_tag};
+    const Address entry_list_address{monitor, ObjectMonitor::entry_list_offset() - monitor_tag};
     const Address owner_address{monitor, ObjectMonitor::owner_offset() - monitor_tag};
 
     Label recursive;
@@ -785,9 +781,8 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax,
     // StoreLoad achieves this.
     membar(StoreLoad);
 
-    // Check if the entry lists are empty (EntryList first - by convention).
-    movptr(reg_rax, EntryList_address);
-    orptr(reg_rax, cxq_address);
+    // Check if the entry_list is empty.
+    cmpptr(entry_list_address, NULL_WORD);
     jccb(Assembler::zero, unlocked);    // If so we are done.
 
     // Check if there is a successor.

src/hotspot/share/jvmci/vmStructs_jvmci.cpp

@@ -328,8 +328,7 @@
                                                                                                                                      \
   volatile_nonstatic_field(ObjectMonitor,      _owner,                                        int64_t)                               \
   volatile_nonstatic_field(ObjectMonitor,      _recursions,                                   intptr_t)                              \
-  volatile_nonstatic_field(ObjectMonitor,      _cxq,                                          ObjectWaiter*)                         \
-  volatile_nonstatic_field(ObjectMonitor,      _EntryList,                                    ObjectWaiter*)                         \
+  volatile_nonstatic_field(ObjectMonitor,      _entry_list,                                   ObjectWaiter*)                         \
   volatile_nonstatic_field(ObjectMonitor,      _succ,                                         int64_t)                               \
   volatile_nonstatic_field(ObjectMonitor,      _stack_locker,                                 BasicLock*)                            \
                                                                                                                                      \

src/hotspot/share/prims/jvm.cpp

@@ -3766,7 +3766,7 @@ JVM_ENTRY(jobject, JVM_TakeVirtualThreadListToUnblock(JNIEnv* env, jclass ignore
   ParkEvent* parkEvent = ObjectMonitor::vthread_unparker_ParkEvent();
   assert(parkEvent != nullptr, "not initialized");
 
-  OopHandle& list_head = ObjectMonitor::vthread_cxq_head();
+  OopHandle& list_head = ObjectMonitor::vthread_list_head();
   oop vthread_head = nullptr;
   while (true) {
     if (list_head.peek() != nullptr) {