GC update for supporting yield of pinned VirtualThread(JEP491)

runtime/gc_base/ReferenceChainWalker.cpp

@@ -32,6 +32,9 @@
 
 #include "ClassIteratorClassSlots.hpp"
 #include "ClassIteratorDeclarationOrder.hpp"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "EnvironmentBase.hpp"
 #include "Forge.hpp"
 #include "GCExtensions.hpp"
@@ -418,6 +421,16 @@ MM_ReferenceChainWalker::scanContinuationNativeSlots(J9Object *objectPtr)
 		}
 #endif /* JAVA_SPEC_VERSION >= 19 */
 		GC_VMThreadStackSlotIterator::scanContinuationSlots(currentThread, objectPtr, (void *)&localData, stackSlotIteratorForReferenceChainWalker, false, _trackVisibleStackFrameDepth);
+
+#if JAVA_SPEC_VERSION >= 24
+		J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, objectPtr);
+		GC_ContinuationSlotIterator continuationSlotIterator(currentThread, continuation);
+
+		while (J9Object **slotPtr = continuationSlotIterator.nextSlot()) {
+			doContinuationSlot(slotPtr, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	}
 }
 
@@ -651,6 +664,31 @@ MM_ReferenceChainWalker::doVMClassSlot(J9Class *classPtr)
 	doClassSlot(classPtr, J9GC_ROOT_TYPE_VM_CLASS_SLOT, -1, NULL);
 }
 
+#if JAVA_SPEC_VERSION >= 24
+/**
+ * @todo Provide function documentation
+ */
+void
+MM_ReferenceChainWalker::doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
+{
+	J9Object *slotValue = *slotPtr;
+	/* Only report heap objects */
+	if (isHeapObject(slotValue) && !_heap->objectIsInGap(slotValue)) {
+		switch(continuationSlotIterator->getState()) {
+		case continuationslotiterator_state_monitor_records:
+			doSlot(slotPtr, J9GC_ROOT_TYPE_CONTINUATION_MONITOR, -1, NULL);
+			break;
+		case continuationslotiterator_state_vthread:
+			doSlot(slotPtr, J9GC_ROOT_TYPE_CONTINUATION_SLOT, -1, NULL);
+			break;
+		default:
+			doSlot(slotPtr, J9GC_ROOT_TYPE_UNKNOWN, -1, NULL);
+			break;
+		}
+	}
+}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 /**
  * @todo Provide function documentation
  */

runtime/gc_base/ReferenceChainWalker.hpp

@@ -127,6 +127,9 @@ class MM_ReferenceChainWalker : public MM_RootScanner
 	virtual void doStringTableSlot(J9Object **slotPtr, GC_StringTableIterator *stringTableIterator);
 	virtual void doVMClassSlot(J9Class *classPtr);
 	virtual void doVMThreadSlot(J9Object **slotPtr, GC_VMThreadIterator *vmThreadIterator);
+#if JAVA_SPEC_VERSION >= 24
+	virtual void doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	virtual void doStackSlot(J9Object **slotPtr, void *walkState, const void* stackLocation);
 	virtual void doSlot(J9Object **slotPtr);
 	virtual void doClassSlot(J9Class *classPtr);

runtime/gc_base/RootScanner.cpp

@@ -356,6 +356,20 @@ MM_RootScanner::doClassSlot(J9Class *classPtr)
 	/* ignore class slots by default */
 }
 
+#if JAVA_SPEC_VERSION >= 24
+/**
+ * @todo Provide function documentation
+ */
+void
+MM_RootScanner::doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
+{
+	/* ensure that this isn't a slot pointing into the gap (only matters for split heap VMs) */
+	if (!_extensions->heap->objectIsInGap(*slotPtr)) {
+		doSlot(slotPtr);
+	}
+}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 /**
  * @todo Provide function documentation
  */
@@ -552,6 +566,14 @@ MM_RootScanner::scanOneThread(MM_EnvironmentBase *env, J9VMThread *walkThread, v
 		/* At this point we know that a virtual thread is mounted. We previously scanned its stack,
 		 * and now we will scan carrier's stack, that continuation struct is currently pointing to. */
 		GC_VMThreadStackSlotIterator::scanSlots(currentThread, walkThread, walkThread->currentContinuation, localData, stackSlotIterator, isStackFrameClassWalkNeeded(), _trackVisibleStackFrameDepth);
+#if JAVA_SPEC_VERSION >= 24
+		GC_ContinuationSlotIterator continuationSlotIterator(walkThread, walkThread->currentContinuation);
+
+		while (J9Object **slot = continuationSlotIterator.nextSlot()) {
+			/* do current continuation slot (mounted vthread case, the slot for saved carrier thread) */
+			doContinuationSlot(slot, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	}
 #endif /* JAVA_SPEC_VERSION >= 19 */
 	return false;

runtime/gc_base/RootScanner.hpp

@@ -35,6 +35,9 @@
 
 #include "BaseVirtual.hpp"
 
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "EnvironmentBase.hpp"
 #include "GCExtensions.hpp"
 #include "JVMTIObjectTagTableIterator.hpp"
@@ -605,6 +608,10 @@ class MM_RootScanner : public MM_BaseVirtual
 	virtual void doDoubleMappedObjectSlot(J9Object *objectPtr, struct J9PortVmemIdentifier *identifier);
 #endif /* J9VM_GC_ENABLE_DOUBLE_MAP */
 
+#if JAVA_SPEC_VERSION >= 24
+	virtual void doContinuationSlot(J9Object **slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	/**
 	 * Called for each object stack slot. Subclasses may override.
 	 * 

runtime/gc_glue_java/CompactSchemeFixupObject.cpp

@@ -26,17 +26,21 @@
 #include "objectdescription.h"
 
 #include "CollectorLanguageInterfaceImpl.hpp"
-#include "EnvironmentStandard.hpp"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "Debug.hpp"
+#include "EnvironmentStandard.hpp"
+#include "FlattenedContiguousArrayIterator.hpp"
 #include "HeapRegionIteratorStandard.hpp"
 #include "MixedObjectIterator.hpp"
 #include "ObjectAccessBarrier.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
-#include "VMHelpers.hpp"
 #include "ParallelDispatcher.hpp"
 #include "PointerContiguousArrayIterator.hpp"
-#include "FlattenedContiguousArrayIterator.hpp"
+#include "StackSlotValidator.hpp"
 #include "Task.hpp"
+#include "VMHelpers.hpp"
 
 void
 MM_CompactSchemeFixupObject::fixupMixedObject(omrobjectptr_t objectPtr)
@@ -50,19 +54,42 @@ MM_CompactSchemeFixupObject::fixupMixedObject(omrobjectptr_t objectPtr)
 }
 
 void
-MM_CompactSchemeFixupObject::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slot)
+MM_CompactSchemeFixupObject::doSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr)
+{
+	*slotPtr = _compactScheme->getForwardingPtr(*slotPtr);
+}
+
+#if JAVA_SPEC_VERSION >= 24
+void
+MM_CompactSchemeFixupObject::doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
 {
-	*slot = _compactScheme->getForwardingPtr(*slot);
+	if (isHeapObject(*slotPtr)) {
+		doSlot(env, fromObject, slotPtr);
+	} else if (NULL != *slotPtr) {
+		Assert_MM_true(continuationslotiterator_state_monitor_records == continuationSlotIterator->getState());
+	}
 }
+#endif /* JAVA_SPEC_VERSION >= 24 */
 
+void
+MM_CompactSchemeFixupObject::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation)
+{
+	if (isHeapObject(*slotPtr)) {
+		Assert_MM_validStackSlot(MM_StackSlotValidator(0, *slotPtr, stackLocation, walkState).validate(env));
+		doSlot(env, fromObject, slotPtr);
+	} else if (NULL != *slotPtr) {
+		/* stack object - just validate */
+		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, *slotPtr, stackLocation, walkState).validate(env));
+	}
+}
 /**
  * @todo Provide function documentation
  */
 void
 stackSlotIteratorForCompactScheme(J9JavaVM *javaVM, J9Object **slotPtr, void *localData, J9StackWalkState *walkState, const void *stackLocation)
 {
 	StackIteratorData4CompactSchemeFixupObject *data = (StackIteratorData4CompactSchemeFixupObject *)localData;
-	data->compactSchemeFixupObject->doStackSlot(data->env, data->fromObject, slotPtr);
+	data->compactSchemeFixupObject->doStackSlot(data->env, data->fromObject, slotPtr, walkState, stackLocation);
 }
 
 
@@ -85,6 +112,16 @@ MM_CompactSchemeFixupObject::fixupContinuationNativeSlots(MM_EnvironmentStandard
 		localData.fromObject = objectPtr;
 
 		GC_VMThreadStackSlotIterator::scanContinuationSlots(currentThread, objectPtr, (void *)&localData, stackSlotIteratorForCompactScheme, false, false);
+
+#if JAVA_SPEC_VERSION >= 24
+		J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, objectPtr);
+		GC_ContinuationSlotIterator continuationSlotIterator(currentThread, continuation);
+
+		while (J9Object **slotPtr = continuationSlotIterator.nextSlot()) {
+			doContinuationSlot(env, objectPtr, slotPtr, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 	}
 }
 

runtime/gc_glue_java/CompactSchemeFixupObject.hpp

@@ -29,6 +29,10 @@
 #include "CompactScheme.hpp"
 #include "GCExtensions.hpp"
 
+#if JAVA_SPEC_VERSION >= 24
+class GC_ContinuationSlotIterator;
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
 class MM_CompactSchemeFixupObject {
 public:
 protected:
@@ -38,7 +42,11 @@ class MM_CompactSchemeFixupObject {
 	MM_CompactScheme *_compactScheme;
 public:
 
-	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slot);
+	MMINLINE void doSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr);
+#if JAVA_SPEC_VERSION >= 24
+	void doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
+	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation);
 	/**
 	 * Perform fixup for a single object
 	 * @param env[in] the current thread
@@ -82,6 +90,15 @@ class MM_CompactSchemeFixupObject {
 	 * @param object -- The object of type or subclass of java.util.concurrent.locks.AbstractOwnableSynchronizer.
 	 */
 	MMINLINE void addOwnableSynchronizerObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
+
+	/**
+	 * Determine whether the object pointer is found within the heap proper.
+	 * @return Boolean indicating if the object pointer is within the heap boundaries.
+	 */
+	MMINLINE bool isHeapObject(J9Object *objectPtr)
+	{
+		return (_extensions->heap->getHeapBase() <= (uint8_t *)objectPtr) && (_extensions->heap->getHeapTop() > (uint8_t *)objectPtr);
+	}
 };
 
 typedef struct StackIteratorData4CompactSchemeFixupObject {

runtime/gc_glue_java/ConcurrentMarkingDelegate.cpp

@@ -25,6 +25,9 @@
 #include "AsyncCallbackHandler.hpp"
 #include "ClassLoaderIterator.hpp"
 #include "ConfigurationDelegate.hpp"
+#if JAVA_SPEC_VERSION >= 24
+#include "ContinuationSlotIterator.hpp"
+#endif /* JAVA_SPEC_VERSION >= 24 */
 #include "FinalizeListManager.hpp"
 #include "Heap.hpp"
 #include "HeapRegionDescriptorStandard.hpp"
@@ -35,6 +38,40 @@
 #include "VMInterface.hpp"
 #include "VMThreadListIterator.hpp"
 
+void
+MM_ConcurrentMarkingDelegate::doSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr)
+{
+	_markingScheme->markObject(env, *slotPtr);
+}
+
+#if JAVA_SPEC_VERSION >= 24
+void
+MM_ConcurrentMarkingDelegate::doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator)
+{
+	if (_markingScheme->isHeapObject(*slotPtr) && !env->getExtensions()->heap->objectIsInGap(*slotPtr)) {
+		doSlot(env, slotPtr);
+	} else if (NULL != *slotPtr) {
+		Assert_MM_true(continuationslotiterator_state_monitor_records == continuationSlotIterator->getState());
+	}
+}
+#endif /* JAVA_SPEC_VERSION >= 24 */
+
+void
+MM_ConcurrentMarkingDelegate::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation)
+{
+	omrobjectptr_t object = *slotPtr;
+	if (env->getExtensions()->heap->objectIsInGap(object)) {
+		/* CMVC 136483:  Ensure that the object is not in the gap of a split heap (stack-allocated object) since we can't mark that part of the address space */
+		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, object, stackLocation, walkState).validate(env));
+	} else if (_markingScheme->isHeapObject(object)) {
+		/* heap object - validate and mark */
+		Assert_MM_validStackSlot(MM_StackSlotValidator(0, object, stackLocation, walkState).validate(env));
+		doSlot(env, slotPtr);
+	} else if (NULL != object) {
+		/* stack object - just validate */
+		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, object, stackLocation, walkState).validate(env));
+	}
+}
 /**
  * Concurrents stack slot iterator.
  * Called for each slot in a threads active stack frames which contains a object reference.
@@ -48,19 +85,7 @@ void
 concurrentStackSlotIterator(J9JavaVM *javaVM, omrobjectptr_t *objectIndirect, void *localData, J9StackWalkState *walkState, const void *stackLocation)
 {
 	MM_ConcurrentMarkingDelegate::markSchemeStackIteratorData *data = (MM_ConcurrentMarkingDelegate::markSchemeStackIteratorData *)localData;
-
-	omrobjectptr_t object = *objectIndirect;
-	if (data->env->getExtensions()->heap->objectIsInGap(object)) {
-		/* CMVC 136483:  Ensure that the object is not in the gap of a split heap (stack-allocated object) since we can't mark that part of the address space */
-		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, object, stackLocation, walkState).validate(data->env));
-	} else if (data->markingScheme->isHeapObject(object)) {
-		/* heap object - validate and mark */
-		Assert_MM_validStackSlot(MM_StackSlotValidator(0, object, stackLocation, walkState).validate(data->env));
-		data->markingScheme->markObject(data->env, object);
-	} else if (NULL != object) {
-		/* stack object - just validate */
-		Assert_MM_validStackSlot(MM_StackSlotValidator(MM_StackSlotValidator::NOT_ON_HEAP, object, stackLocation, walkState).validate(data->env));
-	}
+	data->concurrentMarkingDelegate->doStackSlot(data->env, objectIndirect, walkState, stackLocation);
 }
 
 bool
@@ -150,14 +175,21 @@ MM_ConcurrentMarkingDelegate::scanThreadRoots(MM_EnvironmentBase *env)
 	}
 
 	markSchemeStackIteratorData localData;
-	localData.markingScheme = _markingScheme;
+	localData.concurrentMarkingDelegate = this;
 	localData.env = env;
 	/* In a case this thread is a carrier thread, and a virtual thread is mounted, we will scan virtual thread's stack. */
 	GC_VMThreadStackSlotIterator::scanSlots(vmThread, vmThread, (void *)&localData, concurrentStackSlotIterator, true, false);
 
 #if JAVA_SPEC_VERSION >= 19
 	if (NULL != vmThread->currentContinuation) {
 		GC_VMThreadStackSlotIterator::scanSlots(vmThread, vmThread, vmThread->currentContinuation, (void *)&localData, concurrentStackSlotIterator, true, false);
+#if JAVA_SPEC_VERSION >= 24
+		GC_ContinuationSlotIterator continuationSlotIterator(vmThread, vmThread->currentContinuation);
+
+		while (J9Object **slot = continuationSlotIterator.nextSlot()) {
+			doContinuationSlot(env, slot, &continuationSlotIterator);
+		}
+#endif /* JAVA_SPEC_VERSION >= 24 */
 	}
 #endif /* JAVA_SPEC_VERSION >= 19 */
 

runtime/gc_glue_java/ConcurrentMarkingDelegate.hpp

@@ -46,6 +46,9 @@
 #include "ScanClassesMode.hpp"
 #endif /* J9VM_GC_DYNAMIC_CLASS_UNLOADING */
 
+#if JAVA_SPEC_VERSION >= 24
+class GC_ContinuationSlotIterator;
+#endif /* JAVA_SPEC_VERSION >= 24 */
 class GC_VMThreadIterator;
 class MM_ConcurrentGC;
 class MM_MarkingScheme;
@@ -87,7 +90,7 @@ class MM_ConcurrentMarkingDelegate
 	};
 
 	typedef struct markSchemeStackIteratorData {
-		MM_MarkingScheme *markingScheme;
+		MM_ConcurrentMarkingDelegate *concurrentMarkingDelegate;
 		MM_EnvironmentBase *env;
 	} markSchemeStackIteratorData;
 
@@ -114,6 +117,11 @@ class MM_ConcurrentMarkingDelegate
 	 */
 	bool initialize(MM_EnvironmentBase *env, MM_ConcurrentGC *collector);
 
+	MMINLINE void doSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr);
+#if JAVA_SPEC_VERSION >= 24
+	void doContinuationSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, GC_ContinuationSlotIterator *continuationSlotIterator);
+#endif /* JAVA_SPEC_VERSION >= 24 */
+	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t *slotPtr, J9StackWalkState *walkState, const void *stackLocation);
 	/**
 	 * In the case of Weak Consistency platforms we require this method to bring mutator threads to a safe point. A safe
 	 * point is a point at which a GC may occur.