Local GC: Decouple write barrier operations between the GC and EE (do…

…tnet/coreclr#8568)

* Decouple write barrier operations between the GC and EE

* Address code review feedback

* Address code review feedback

* Repair the standalone GC build


Commit migrated from dotnet/coreclr@04d6bd1

src/coreclr/src/gc/env/gcenv.ee.h

@@ -65,6 +65,7 @@ class GCToEEInterface
  static void DiagWalkSurvivors(void* gcContext);
  static void DiagWalkLOHSurvivors(void* gcContext);
  static void DiagWalkBGCSurvivors(void* gcContext);
+ static void StompWriteBarrier(WriteBarrierParameters* args);
 };
 
 #endif // __GCENV_EE_H__

src/coreclr/src/gc/gc.h

@@ -137,6 +137,12 @@ class DacHeapWalker;
 
 #define MP_LOCKS
 
+extern "C" uint32_t* g_gc_card_table;
+extern "C" uint8_t* g_gc_lowest_address;
+extern "C" uint8_t* g_gc_highest_address;
+extern "C" uint8_t* g_gc_ephemeral_low;
+extern "C" uint8_t* g_gc_ephemeral_high;
+
 namespace WKS {
  ::IGCHeapInternal* CreateGCHeap();
  class GCHeap;

src/coreclr/src/gc/gccommon.cpp

@@ -26,28 +26,24 @@ IGCHeapInternal* g_theGCHeap;
 IGCToCLR* g_theGCToCLR;
 #endif // FEATURE_STANDALONE_GC
 
-/* global versions of the card table and brick table */ 
-GPTR_IMPL(uint32_t,g_card_table);
-
-/* absolute bounds of the GC memory */
-GPTR_IMPL_INIT(uint8_t,g_lowest_address,0);
-GPTR_IMPL_INIT(uint8_t,g_highest_address,0);
-
 #ifdef GC_CONFIG_DRIVEN
 GARY_IMPL(size_t, gc_global_mechanisms, MAX_GLOBAL_GC_MECHANISMS_COUNT);
 #endif //GC_CONFIG_DRIVEN
 
 #ifndef DACCESS_COMPILE
 
-uint8_t* g_ephemeral_low = (uint8_t*)1;
-uint8_t* g_ephemeral_high = (uint8_t*)~0;
-
 #ifdef WRITE_BARRIER_CHECK
 uint8_t* g_GCShadow;
 uint8_t* g_GCShadowEnd;
 uint8_t* g_shadow_lowest_address = NULL;
 #endif
 
+uint32_t* g_gc_card_table;
+uint8_t* g_gc_lowest_address = 0;
+uint8_t* g_gc_highest_address = 0;
+uint8_t* g_gc_ephemeral_low = (uint8_t*)1;
+uint8_t* g_gc_ephemeral_high = (uint8_t*)~0;
+
 VOLATILE(int32_t) m_GCLock = -1;
 
 #ifdef GC_CONFIG_DRIVEN

src/coreclr/src/gc/gcenv.ee.standalone.inl

@@ -166,4 +166,11 @@ inline void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext)
  assert(g_theGCToCLR != nullptr);
  return g_theGCToCLR->DiagWalkBGCSurvivors(gcContext);
 }
+
+inline void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
+{
+ assert(g_theGCToCLR != nullptr);
+ g_theGCToCLR->StompWriteBarrier(args);
+}
+
 #endif // __GCTOENV_EE_STANDALONE_INL__

src/coreclr/src/gc/gcinterface.ee.h

@@ -123,6 +123,11 @@ class IGCToCLR {
  // At the end of a background GC, gives the diagnostics code a chance to run.
  virtual
  void DiagWalkBGCSurvivors(void* gcContext) = 0;
+
+ // Informs the EE of changes to the location of the card table, potentially updating the write
+ // barrier if it needs to be updated.
+ virtual
+ void StompWriteBarrier(WriteBarrierParameters* args) = 0;
 };
 
 #endif // _GCINTERFACE_EE_H_

src/coreclr/src/gc/gcinterface.h

@@ -41,6 +41,57 @@ typedef enum
  walk_for_loh = 3
 } walk_surv_type;
 
+// Different operations that can be done by GCToEEInterface::StompWriteBarrier
+enum class WriteBarrierOp
+{
+ StompResize,
+ StompEphemeral,
+ Initialize
+};
+
+// Arguments to GCToEEInterface::StompWriteBarrier
+struct WriteBarrierParameters
+{
+ // The operation that StompWriteBarrier will perform.
+ WriteBarrierOp operation;
+
+ // Whether or not the runtime is currently suspended. If it is not,
+ // the EE will need to suspend it before bashing the write barrier.
+ // Used for all operations.
+ bool is_runtime_suspended;
+
+ // Whether or not the GC has moved the ephemeral generation to no longer
+ // be at the top of the heap. When the ephemeral generation is at the top
+ // of the heap, and the write barrier observes that a pointer is greater than
+ // g_ephemeral_low, it does not need to check that the pointer is less than
+ // g_ephemeral_high because there is nothing in the GC heap above the ephemeral
+ // generation. When this is not the case, however, the GC must inform the EE
+ // so that the EE can switch to a write barrier that checks that a pointer
+ // is both greater than g_ephemeral_low and less than g_ephemeral_high.
+ // Used for WriteBarrierOp::StompResize.
+ bool requires_upper_bounds_check;
+
+ // The new card table location. May or may not be the same as the previous
+ // card table. Used for WriteBarrierOp::Initialize and WriteBarrierOp::StompResize.
+ uint32_t* card_table;
+
+ // The heap's new low boundary. May or may not be the same as the previous
+ // value. Used for WriteBarrierOp::Initialize and WriteBarrierOp::StompResize.
+ uint8_t* lowest_address;
+
+ // The heap's new high boundary. May or may not be the same as the previous
+ // value. Used for WriteBarrierOp::Initialize and WriteBarrierOp::StompResize.
+ uint8_t* highest_address;
+
+ // The new start of the ephemeral generation. 
+ // Used for WriteBarrierOp::StompEphemeral.
+ uint8_t* ephemeral_lo;
+
+ // The new end of the ephemeral generation.
+ // Used for WriteBarrierOp::StompEphemeral.
+ uint8_t* ephemeral_hi;
+};
+
 #include "gcinterface.ee.h"
 
 // The allocation context must be known to the VM for use in the allocation
@@ -110,19 +161,6 @@ IGCHeap* InitializeGarbageCollector(IGCToCLR* clrToGC);
 // and the heap is actually recated.
 void InitializeHeapType(bool bServerHeap);
 
-#ifndef DACCESS_COMPILE
-extern "C" {
-#endif // !DACCESS_COMPILE
-GPTR_DECL(uint8_t,g_lowest_address);
-GPTR_DECL(uint8_t,g_highest_address);
-GPTR_DECL(uint32_t,g_card_table);
-#ifndef DACCESS_COMPILE
-} 
-#endif // !DACCESS_COMPILE
-
-extern "C" uint8_t* g_ephemeral_low;
-extern "C" uint8_t* g_ephemeral_high;
-
 #ifdef WRITE_BARRIER_CHECK
 //always defined, but should be 0 in Server GC
 extern uint8_t* g_GCShadow;

src/coreclr/src/gc/gcpriv.h

@@ -4319,9 +4319,6 @@ dynamic_data* gc_heap::dynamic_data_of (int gen_number)
  return &dynamic_data_table [ gen_number ];
 }
 
-extern "C" uint8_t* g_ephemeral_low;
-extern "C" uint8_t* g_ephemeral_high;
-
 #define card_word_width ((size_t)32)
 
 //

src/coreclr/src/gc/sample/GCSample.cpp

@@ -91,14 +91,14 @@ inline void ErectWriteBarrier(Object ** dst, Object * ref)
 {
  // if the dst is outside of the heap (unboxed value classes) then we
  // simply exit
- if (((uint8_t*)dst < g_lowest_address) || ((uint8_t*)dst >= g_highest_address))
+ if (((uint8_t*)dst < g_gc_lowest_address) || ((uint8_t*)dst >= g_gc_highest_address))
  return;
 
- if((uint8_t*)ref >= g_ephemeral_low && (uint8_t*)ref < g_ephemeral_high)
+ if((uint8_t*)ref >= g_gc_ephemeral_low && (uint8_t*)ref < g_gc_ephemeral_high)
  {
  // volatile is used here to prevent fetch of g_card_table from being reordered 
  // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
- uint8_t* pCardByte = (uint8_t *)*(volatile uint8_t **)(&g_card_table) + card_byte((uint8_t *)dst);
+ uint8_t* pCardByte = (uint8_t *)*(volatile uint8_t **)(&g_gc_card_table) + card_byte((uint8_t *)dst);
  if(*pCardByte != 0xFF)
  *pCardByte = 0xFF;
  }

src/coreclr/src/gc/sample/gcenv.ee.cpp

@@ -249,6 +249,10 @@ void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext)
 {
 }
 
+void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
+{
+}
+
 void FinalizerThread::EnableFinalization()
 {
  // Signal to finalizer thread that there are objects to finalize
@@ -266,14 +270,6 @@ bool IsGCSpecialThread()
  return false;
 }
 
-void StompWriteBarrierEphemeral(bool /* isRuntimeSuspended */)
-{
-}
-
-void StompWriteBarrierResize(bool /* isRuntimeSuspended */, bool /*bReqUpperBoundsCheck*/)
-{
-}
-
 bool IsGCThread()
 {
  return false;

src/coreclr/src/vm/gcenv.ee.cpp

@@ -1214,3 +1214,57 @@ void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext)
 #endif //GC_PROFILING || FEATURE_EVENT_TRACE
 }
 
+void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
+{
+ assert(args != nullptr);
+ switch (args->operation)
+ {
+ case WriteBarrierOp::StompResize:
+ // StompResize requires a new card table, a new lowest address, and
+ // a new highest address
+ assert(args->card_table != nullptr);
+ assert(args->lowest_address != nullptr);
+ assert(args->highest_address != nullptr);
+ g_card_table = args->card_table;
+ ::StompWriteBarrierResize(args->is_runtime_suspended, args->requires_upper_bounds_check);
+
+ // We need to make sure that other threads executing checked write barriers
+ // will see the g_card_table update before g_lowest/highest_address updates.
+ // Otherwise, the checked write barrier may AV accessing the old card table
+ // with address that it does not cover. Write barriers access card table
+ // without memory barriers for performance reasons, so we need to flush
+ // the store buffers here.
+ FlushProcessWriteBuffers();
+
+ g_lowest_address = args->lowest_address;
+ VolatileStore(&g_highest_address, args->highest_address);
+ return;
+ case WriteBarrierOp::StompEphemeral:
+ // StompEphemeral requires a new ephemeral low and a new ephemeral high
+ assert(args->ephemeral_lo != nullptr);
+ assert(args->ephemeral_hi != nullptr);
+ g_ephemeral_low = args->ephemeral_lo;
+ g_ephemeral_high = args->ephemeral_hi;
+ ::StompWriteBarrierEphemeral(args->is_runtime_suspended);
+ return;
+ case WriteBarrierOp::Initialize:
+ // This operation should only be invoked once, upon initialization.
+ assert(g_card_table == nullptr);
+ assert(g_lowest_address == nullptr);
+ assert(g_highest_address == nullptr);
+ assert(args->card_table != nullptr);
+ assert(args->lowest_address != nullptr);
+ assert(args->highest_address != nullptr);
+ assert(args->is_runtime_suspended && "the runtime must be suspended here!");
+ assert(!args->requires_upper_bounds_check && "the ephemeral generation must be at the top of the heap!");
+
+ g_card_table = args->card_table;
+ FlushProcessWriteBuffers();
+ g_lowest_address = args->lowest_address;
+ VolatileStore(&g_highest_address, args->highest_address);
+ ::StompWriteBarrierResize(true, false);
+ return;
+ default:
+ assert(!"unknown WriteBarrierOp enum");
+ }
+}

src/coreclr/src/vm/gcenv.ee.h

@@ -41,6 +41,7 @@ class GCToEEInterface : public IGCToCLR {
  void DiagWalkSurvivors(void* gcContext);
  void DiagWalkLOHSurvivors(void* gcContext);
  void DiagWalkBGCSurvivors(void* gcContext);
+ void StompWriteBarrier(WriteBarrierParameters* args);
 };
 
 #endif // FEATURE_STANDALONE_GC

src/coreclr/src/vm/gcheaputilities.cpp

@@ -5,5 +5,15 @@
 #include "common.h"
 #include "gcheaputilities.h"
 
+// These globals are variables used within the GC and maintained
+// by the EE for use in write barriers. It is the responsibility
+// of the GC to communicate updates to these globals to the EE through
+// GCToEEInterface::StompWriteBarrierResize and GCToEEInterface::StompWriteBarrierEphemeral.
+GPTR_IMPL_INIT(uint32_t, g_card_table, nullptr);
+GPTR_IMPL_INIT(uint8_t, g_lowest_address, nullptr);
+GPTR_IMPL_INIT(uint8_t, g_highest_address, nullptr);
+uint8_t* g_ephemeral_low = (uint8_t*)1;
+uint8_t* g_ephemeral_high = (uint8_t*)~0;
+
 // This is the global GC heap, maintained by the VM.
 GPTR_IMPL(IGCHeap, g_pGCHeap);

src/coreclr/src/vm/gcheaputilities.h

@@ -113,4 +113,17 @@ class GCHeapUtilities {
  GCHeapUtilities() = delete;
 };
 
+#ifndef DACCESS_COMPILE
+extern "C" {
+#endif // !DACCESS_COMPILE
+GPTR_DECL(uint8_t,g_lowest_address);
+GPTR_DECL(uint8_t,g_highest_address);
+GPTR_DECL(uint32_t,g_card_table);
+#ifndef DACCESS_COMPILE
+} 
+#endif // !DACCESS_COMPILE
+
+extern "C" uint8_t* g_ephemeral_low;
+extern "C" uint8_t* g_ephemeral_high;
+
 #endif // _GCHEAPUTILITIES_H_