8370345: Parallel: Rework TLAB accounting in MutableNUMASpace

src/hotspot/os/linux/os_linux.cpp

@@ -846,10 +846,6 @@ static void *thread_native_entry(Thread *thread) {
 
   osthread->set_thread_id(checked_cast<pid_t>(os::current_thread_id()));
 
-  if (UseNUMA) {
-    thread->update_lgrp_id();
-  }
-
   // initialize signal mask for this thread
   PosixSignals::hotspot_sigmask(thread);
 
@@ -1175,10 +1171,6 @@ bool os::create_attached_thread(JavaThread* thread) {
 
   thread->set_osthread(osthread);
 
-  if (UseNUMA) {
-    thread->update_lgrp_id();
-  }
-
   if (os::is_primordial_thread()) {
     // If current thread is primordial thread, its stack is mapped on demand,
     // see notes about MAP_GROWSDOWN. Here we try to force kernel to map

src/hotspot/share/gc/parallel/mutableNUMASpace.cpp

@@ -36,6 +36,7 @@
 #include "runtime/os.inline.hpp"
 #include "runtime/threadSMR.hpp"
 #include "utilities/align.hpp"
+#include "utilities/globalDefinitions.hpp"
 
 MutableNUMASpace::MutableNUMASpace(size_t page_size) : MutableSpace(page_size) {
   _lgrp_spaces = new (mtGC) GrowableArray<LGRPSpace*>(0, mtGC);
@@ -94,47 +95,42 @@ void MutableNUMASpace::ensure_parsability() {
 
 size_t MutableNUMASpace::used_in_words() const {
   size_t s = 0;
-  for (int i = 0; i < lgrp_spaces()->length(); i++) {
-    s += lgrp_spaces()->at(i)->space()->used_in_words();
+  for (LGRPSpace* ls : *lgrp_spaces()) {
+    s += ls->space()->used_in_words();
   }
   return s;
 }
 
 size_t MutableNUMASpace::free_in_words() const {
   size_t s = 0;
-  for (int i = 0; i < lgrp_spaces()->length(); i++) {
-    s += lgrp_spaces()->at(i)->space()->free_in_words();
+  for (LGRPSpace* ls : *lgrp_spaces()) {
+    s += ls->space()->free_in_words();
   }
   return s;
 }
 
-MutableNUMASpace::LGRPSpace *MutableNUMASpace::lgrp_space_for_thread(Thread* thr) const {
-  guarantee(thr != nullptr, "No thread");
-
-  int lgrp_id = thr->lgrp_id();
-  assert(lgrp_id != -1, "lgrp_id must be set during thread creation");
-
-  int lgrp_spaces_index = lgrp_spaces()->find_if([&](LGRPSpace* space) {
-    return space->lgrp_id() == (uint)lgrp_id;
-  });
-
-  if (lgrp_spaces_index == -1) {
-    // Running on a CPU with no memory; pick another CPU based on %.
-    lgrp_spaces_index = lgrp_id % lgrp_spaces()->length();
+size_t MutableNUMASpace::tlab_capacity(Thread *ignored) const {
+  size_t s = 0;
+  for (LGRPSpace* ls : *lgrp_spaces()) {
+    s += ls->space()->capacity_in_bytes();
   }
-  return lgrp_spaces()->at(lgrp_spaces_index);
-}
-
-size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
-  return lgrp_space_for_thread(thr)->space()->capacity_in_bytes();
+  return s / (size_t)lgrp_spaces()->length();
 }
 
-size_t MutableNUMASpace::tlab_used(Thread *thr) const {
-  return lgrp_space_for_thread(thr)->space()->used_in_bytes();
+size_t MutableNUMASpace::tlab_used(Thread *ignored) const {
+  size_t s = 0;
+  for (LGRPSpace* ls : *lgrp_spaces()) {
+    s += ls->space()->used_in_bytes();
+  }
+  return s / (size_t)lgrp_spaces()->length();
 }
 
-size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
-  return lgrp_space_for_thread(thr)->space()->free_in_bytes();
+size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *ignored) const {
+  size_t s = 0;
+  for (LGRPSpace* ls : *lgrp_spaces()) {
+    s += ls->space()->free_in_bytes();
+  }
+  return align_down(s / (size_t)lgrp_spaces()->length(), MinObjAlignmentInBytes);
 }
 
 // Bias region towards the first-touching lgrp. Set the right page sizes.
@@ -442,13 +438,22 @@ void MutableNUMASpace::clear(bool mangle_space) {
   }
 }
 
-HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
-  Thread *thr = Thread::current();
+MutableNUMASpace::LGRPSpace *MutableNUMASpace::lgrp_space_for_current_thread() const {
+  const int lgrp_id = os::numa_get_group_id();
+  int lgrp_spaces_index = lgrp_spaces()->find_if([&](LGRPSpace* space) {
+    return space->lgrp_id() == (uint)lgrp_id;
+  });
 
-  // Update the locality group to match where the thread actually is.
-  thr->update_lgrp_id();
+  if (lgrp_spaces_index == -1) {
+    // Running on a CPU with no memory; pick another CPU based on %.
+    lgrp_spaces_index = lgrp_id % lgrp_spaces()->length();
+  }
 
-  LGRPSpace *ls = lgrp_space_for_thread(thr);
+  return lgrp_spaces()->at(lgrp_spaces_index);
+}
+
+HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
+  LGRPSpace *ls = lgrp_space_for_current_thread();
   MutableSpace *s = ls->space();
   HeapWord *p = s->cas_allocate(size);
   if (p != nullptr) {

src/hotspot/share/gc/parallel/mutableNUMASpace.hpp

@@ -140,7 +140,7 @@ class MutableNUMASpace : public MutableSpace {
   void select_tails(MemRegion new_region, MemRegion intersection,
                     MemRegion* bottom_region, MemRegion *top_region);
 
-  LGRPSpace *lgrp_space_for_thread(Thread *thr) const;
+  LGRPSpace *lgrp_space_for_current_thread() const;
 
 public:
   GrowableArray<LGRPSpace*>* lgrp_spaces() const     { return _lgrp_spaces;       }
@@ -166,9 +166,9 @@ class MutableNUMASpace : public MutableSpace {
   virtual size_t used_in_words() const;
   virtual size_t free_in_words() const;
 
-  virtual size_t tlab_capacity(Thread* thr) const;
-  virtual size_t tlab_used(Thread* thr) const;
-  virtual size_t unsafe_max_tlab_alloc(Thread* thr) const;
+  virtual size_t tlab_capacity(Thread* ignored) const;
+  virtual size_t tlab_used(Thread* ignored) const;
+  virtual size_t unsafe_max_tlab_alloc(Thread* ignored) const;
 
   // Allocation (return null if full)
   virtual HeapWord* cas_allocate(size_t word_size);

src/hotspot/share/runtime/thread.cpp

@@ -65,7 +65,6 @@ Thread::Thread(MemTag mem_tag) {
   // stack and get_thread
   set_stack_base(nullptr);
   set_stack_size(0);
-  _lgrp_id = -1;
   DEBUG_ONLY(clear_suspendible_thread();)
   DEBUG_ONLY(clear_indirectly_suspendible_thread();)
   DEBUG_ONLY(clear_indirectly_safepoint_thread();)

src/hotspot/share/runtime/thread.hpp

@@ -522,7 +522,6 @@ class Thread: public ThreadShadow {
   // Support for stack overflow handling, get_thread, etc.
   address          _stack_base;
   size_t           _stack_size;
-  int              _lgrp_id;
 
  public:
   // Stack overflow support
@@ -537,9 +536,6 @@ class Thread: public ThreadShadow {
   void    register_thread_stack_with_NMT();
   void    unregister_thread_stack_with_NMT();
 
-  int     lgrp_id() const  { return _lgrp_id; }
-  void    update_lgrp_id() { _lgrp_id = os::numa_get_group_id(); }
-
   // Printing
   void print_on(outputStream* st, bool print_extended_info) const;
   virtual void print_on(outputStream* st) const { print_on(st, false); }