Merge pull request #15603 from LinHu2016/GC_SCAN_Continuation_Loom

Loom: GC Scan Continuation Object
eclipse-openj9 · Sep 2, 2022 · 00f03c2 · 00f03c2
2 parents fad2763 + 69c8e12
commit 00f03c2
Show file tree

Hide file tree

Showing 46 changed files with 913 additions and 54 deletions.
diff --git a/runtime/compiler/env/J9ClassEnv.cpp b/runtime/compiler/env/J9ClassEnv.cpp
@@ -87,7 +87,8 @@ J9::ClassEnv::isClassSpecialForStackAllocation(TR_OpaqueClassBlock * clazz)
    const UDATA mask = (J9AccClassReferenceWeak |
                        J9AccClassReferenceSoft |
                        J9AccClassFinalizeNeeded |
-                       J9AccClassOwnableSynchronizer);
+                       J9AccClassOwnableSynchronizer |
+                       J9AccClassContinuation );
 
 #if defined(J9VM_OPT_JITSERVER)
    if (auto stream = TR::CompilationInfo::getStream())

diff --git a/runtime/compiler/env/VMJ9.cpp b/runtime/compiler/env/VMJ9.cpp
@@ -1588,7 +1588,7 @@ UDATA TR_J9VMBase::constClassFlagsPublic()      {return J9AccPublic;}
 
 int32_t TR_J9VMBase::getFlagValueForPrimitiveTypeCheck()        {return J9AccClassInternalPrimitiveType;}
 int32_t TR_J9VMBase::getFlagValueForArrayCheck()                {return J9AccClassArray;}
-int32_t TR_J9VMBase::getFlagValueForFinalizerCheck()            {return J9AccClassFinalizeNeeded | J9AccClassOwnableSynchronizer;}
+int32_t TR_J9VMBase::getFlagValueForFinalizerCheck()            {return J9AccClassFinalizeNeeded | J9AccClassOwnableSynchronizer | J9AccClassContinuation;}
 
 
 UDATA TR_J9VMBase::getGCForwardingPointerOffset()               {
@@ -6497,6 +6497,13 @@ TR_J9VMBase::isOwnableSyncClass(TR_OpaqueClassBlock *clazz)
    return ((J9CLASS_FLAGS(j9class) & J9AccClassOwnableSynchronizer) != 0);
    }
 
+bool
+TR_J9VMBase::isContinuationClass(TR_OpaqueClassBlock *clazz)
+   {
+   J9Class* j9class = TR::Compiler->cls.convertClassOffsetToClassPtr(clazz);
+   return ((J9CLASS_FLAGS(j9class) & J9AccClassContinuation) != 0);
+   }
+
 const char *
 TR_J9VMBase::getByteCodeName(uint8_t opcode)
    {
@@ -6831,7 +6838,7 @@ TR_J9VM::isPublicClass(TR_OpaqueClassBlock * clazz)
 bool
 TR_J9VMBase::hasFinalizer(TR_OpaqueClassBlock * classPointer)
    {
-   return (J9CLASS_FLAGS(TR::Compiler->cls.convertClassOffsetToClassPtr(classPointer)) & (J9AccClassFinalizeNeeded | J9AccClassOwnableSynchronizer)) != 0;
+   return (J9CLASS_FLAGS(TR::Compiler->cls.convertClassOffsetToClassPtr(classPointer)) & (J9AccClassFinalizeNeeded | J9AccClassOwnableSynchronizer | J9AccClassContinuation)) != 0;
    }
 
 uintptr_t

diff --git a/runtime/compiler/env/VMJ9.h b/runtime/compiler/env/VMJ9.h
@@ -1289,6 +1289,8 @@ class TR_J9VMBase : public TR_FrontEnd
    virtual bool isOwnableSyncClass(TR_OpaqueClassBlock *clazz);
    const char *getJ9MonitorName(J9ThreadMonitor* monitor);
 
+   virtual bool  isContinuationClass(TR_OpaqueClassBlock *clazz);
+
    virtual TR_J9SharedCache *sharedCache() { return _sharedCache; }
    virtual void              freeSharedCache();
 

diff --git a/runtime/compiler/env/VMJ9Server.cpp b/runtime/compiler/env/VMJ9Server.cpp
@@ -1397,7 +1397,7 @@ TR_J9ServerVM::hasFinalizer(TR_OpaqueClassBlock *clazz)
    JITServer::ServerStream *stream = _compInfoPT->getMethodBeingCompiled()->_stream;
    JITServerHelpers::getAndCacheRAMClassInfo((J9Class *)clazz, _compInfoPT->getClientData(), stream, JITServerHelpers::CLASSINFO_CLASS_DEPTH_AND_FLAGS, (void *)&classDepthAndFlags);
 
-   return ((classDepthAndFlags & (J9AccClassFinalizeNeeded | J9AccClassOwnableSynchronizer)) != 0);
+   return ((classDepthAndFlags & (J9AccClassFinalizeNeeded | J9AccClassOwnableSynchronizer | J9AccClassContinuation)) != 0);
    }
 
 uintptr_t
@@ -1590,6 +1590,16 @@ TR_J9ServerVM::isOwnableSyncClass(TR_OpaqueClassBlock *clazz)
    return ((classDepthAndFlags & J9AccClassOwnableSynchronizer) != 0);
    }
 
+bool
+TR_J9ServerVM::isContinuationClass(TR_OpaqueClassBlock *clazz)
+   {
+   uintptr_t classDepthAndFlags = 0;
+   JITServer::ServerStream *stream = _compInfoPT->getMethodBeingCompiled()->_stream;
+   JITServerHelpers::getAndCacheRAMClassInfo((J9Class *)clazz, _compInfoPT->getClientData(), stream, JITServerHelpers::CLASSINFO_CLASS_DEPTH_AND_FLAGS, (void *)&classDepthAndFlags);
+
+   return ((classDepthAndFlags & J9AccClassContinuation) != 0);
+   }
+
 TR_OpaqueClassBlock *
 TR_J9ServerVM::getClassFromMethodBlock(TR_OpaqueMethodBlock *method)
    {

diff --git a/runtime/compiler/env/VMJ9Server.hpp b/runtime/compiler/env/VMJ9Server.hpp
@@ -171,6 +171,7 @@ class TR_J9ServerVM: public TR_J9VM
    virtual void revertToInterpreted(TR_OpaqueMethodBlock *method) override;
    virtual void * getLocationOfClassLoaderObjectPointer(TR_OpaqueClassBlock *clazz) override;
    virtual bool isOwnableSyncClass(TR_OpaqueClassBlock *clazz) override;
+   virtual bool isContinuationClass(TR_OpaqueClassBlock *clazz) override;
    virtual TR_OpaqueClassBlock * getClassFromMethodBlock(TR_OpaqueMethodBlock *method) override;
    virtual U_8 * fetchMethodExtendedFlagsPointer(J9Method *method) override;
    virtual void * getStaticHookAddress(int32_t event) override;

diff --git a/runtime/compiler/net/CommunicationStream.hpp b/runtime/compiler/net/CommunicationStream.hpp
@@ -97,7 +97,7 @@ class CommunicationStream
    ClientMessage _cMsg;
 
    static const uint8_t MAJOR_NUMBER = 1;
-   static const uint16_t MINOR_NUMBER = 40;
+   static const uint16_t MINOR_NUMBER = 41;
    static const uint8_t PATCH_NUMBER = 0;
    static uint32_t CONFIGURATION_FLAGS;
 

diff --git a/runtime/gc_api/HeapIteratorAPI.cpp b/runtime/gc_api/HeapIteratorAPI.cpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2019 IBM Corp. and others
+ * Copyright (c) 1991, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -402,12 +402,12 @@ j9mm_iterate_object_slots(
 	case GC_ObjectModel::SCAN_ATOMIC_MARKABLE_REFERENCE_OBJECT:
 	case GC_ObjectModel::SCAN_MIXED_OBJECT:
 	case GC_ObjectModel::SCAN_OWNABLESYNCHRONIZER_OBJECT:
+	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
 	case GC_ObjectModel::SCAN_CLASS_OBJECT:
 	case GC_ObjectModel::SCAN_CLASSLOADER_OBJECT:
 	case GC_ObjectModel::SCAN_REFERENCE_MIXED_OBJECT:
 		returnCode = iterateMixedObjectSlots(javaVM, objectPtr, object, flags, func, userData);
 		break;
-
 	case GC_ObjectModel::SCAN_POINTER_ARRAY_OBJECT:
 		returnCode = iterateArrayObjectSlots(javaVM, objectPtr, object, flags, func, userData);
 		if (JVMTI_ITERATION_CONTINUE == returnCode) {

diff --git a/runtime/gc_base/ObjectAccessBarrier.cpp b/runtime/gc_base/ObjectAccessBarrier.cpp
@@ -92,7 +92,10 @@ MM_ObjectAccessBarrier::initialize(MM_EnvironmentBase *env)
 	if (0 != vm->internalVMFunctions->addHiddenInstanceField(vm, "java/util/concurrent/locks/AbstractOwnableSynchronizer", "ownableSynchronizerLink", refSignature, &_ownableSynchronizerLinkOffset)) {
 		return false;
 	}
-
+	/* request an extra slot in java/lang/VirtualThread$VThreadContinuation which we will use to maintain linked lists of continuation objects */
+	if (0 != vm->internalVMFunctions->addHiddenInstanceField(vm, "jdk/internal/vm/Continuation", "continuationLink", refSignature, &_continuationLinkOffset)) {
+		return false;
+	}
 	return true;
 }
 
@@ -2262,6 +2265,18 @@ MM_ObjectAccessBarrier::setOwnableSynchronizerLink(j9object_t object, j9object_t
 	slot.writeReferenceToSlot(value);
 }
 
+void
+MM_ObjectAccessBarrier::setContinuationLink(j9object_t object, j9object_t value)
+{
+	Assert_MM_true(NULL != object);
+	UDATA linkOffset = _continuationLinkOffset;
+	/* offset will be UDATA_MAX until Continuation is loaded */
+	Assert_MM_true(UDATA_MAX != linkOffset);
+	fj9object_t *continuationLink = (fj9object_t*)((UDATA)object + linkOffset);
+	GC_SlotObject slot(_extensions->getOmrVM(), continuationLink);
+	slot.writeReferenceToSlot(value);
+}
+
 void
 MM_ObjectAccessBarrier::printNativeMethod(J9VMThread* vmThread)
 {

diff --git a/runtime/gc_base/ObjectAccessBarrier.hpp b/runtime/gc_base/ObjectAccessBarrier.hpp
@@ -60,6 +60,7 @@ class MM_ObjectAccessBarrier : public MM_BaseVirtual
 #endif /* defined(OMR_GC_COMPRESSED_POINTERS) */
 	UDATA _referenceLinkOffset; /** Offset within java/lang/ref/Reference of the reference link field */
 	UDATA _ownableSynchronizerLinkOffset; /** Offset within java/util/concurrent/locks/AbstractOwnableSynchronizer of the ownable synchronizer link field */
+	UDATA _continuationLinkOffset; /** Offset within java/lang/VirtualThread$VThreadContinuation (jdk/internal/vm/Continuation) of the continuation link field */
 public:
 
 	/* member function */
@@ -490,6 +491,26 @@ class MM_ObjectAccessBarrier : public MM_BaseVirtual
 		return slot.readReferenceFromSlot();
 	}
 
+	/**
+	 * Set the continuationLink link field of the specified reference object to value.
+	 * @param object the object to modify
+	 * @param value the value to store into the object's reference link field
+	 */
+	void setContinuationLink(j9object_t object, j9object_t value);
+
+	/**
+	 * Fetch the continuationLink link field of the specified reference object.
+	 * @param object the object to read
+	 * @return the value stored in the object's reference link field
+	 */
+	j9object_t getContinuationLink(j9object_t object)
+	{
+		UDATA linkOffset = _continuationLinkOffset;
+		fj9object_t *continuationLink = (fj9object_t*)((UDATA)object + linkOffset);
+		GC_SlotObject slot(_extensions->getOmrVM(), continuationLink);
+		return slot.readReferenceFromSlot();
+	}
+
 	/**
 	 * Implementation of the JNI GetPrimitiveArrayCritical API.
 	 * See the JNI spec for full details.
@@ -592,6 +613,7 @@ class MM_ObjectAccessBarrier : public MM_BaseVirtual
 #endif /* defined(OMR_GC_COMPRESSED_POINTERS) */
 		, _referenceLinkOffset(UDATA_MAX)
 		, _ownableSynchronizerLinkOffset(UDATA_MAX)
+		, _continuationLinkOffset(UDATA_MAX)
 	{
 		_typeId = __FUNCTION__;
 	}

diff --git a/runtime/gc_base/ReferenceChainWalker.cpp b/runtime/gc_base/ReferenceChainWalker.cpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2021 IBM Corp. and others
+ * Copyright (c) 1991, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -351,6 +351,7 @@ MM_ReferenceChainWalker::scanObject(J9Object *objectPtr)
 	case GC_ObjectModel::SCAN_ATOMIC_MARKABLE_REFERENCE_OBJECT:
 	case GC_ObjectModel::SCAN_MIXED_OBJECT:
 	case GC_ObjectModel::SCAN_OWNABLESYNCHRONIZER_OBJECT:
+	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
 	case GC_ObjectModel::SCAN_CLASS_OBJECT:
 	case GC_ObjectModel::SCAN_CLASSLOADER_OBJECT:
 		scanMixedObject(objectPtr);

diff --git a/runtime/gc_glue_java/CMakeLists.txt b/runtime/gc_glue_java/CMakeLists.txt
@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2017, 2020 IBM Corp. and others
+# Copyright (c) 2017, 2022 IBM Corp. and others
 #
 # This program and the accompanying materials are made available under
 # the terms of the Eclipse Public License 2.0 which accompanies this
@@ -31,6 +31,7 @@ set(j9vm_gc_glue_sources
 	${CMAKE_CURRENT_SOURCE_DIR}/ConcurrentSafepointCallbackJava.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/EnvironmentDelegate.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/GlobalCollectorDelegate.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/HeapWalkerDelegate.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/JNICriticalRegion.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/MarkingDelegate.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/MarkingSchemeRootClearer.cpp

diff --git a/runtime/gc_glue_java/CompactDelegate.cpp b/runtime/gc_glue_java/CompactDelegate.cpp
@@ -1,6 +1,6 @@
 
 /*******************************************************************************
- * Copyright (c) 2017, 2020 IBM Corp. and others
+ * Copyright (c) 2017, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -82,6 +82,7 @@ MM_CompactDelegate::verifyHeap(MM_EnvironmentBase *env, MM_MarkMap *markMap)
 			case GC_ObjectModel::SCAN_ATOMIC_MARKABLE_REFERENCE_OBJECT:
 			case GC_ObjectModel::SCAN_MIXED_OBJECT:
 			case GC_ObjectModel::SCAN_OWNABLESYNCHRONIZER_OBJECT:
+			case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
 			case GC_ObjectModel::SCAN_CLASS_OBJECT:
 			case GC_ObjectModel::SCAN_CLASSLOADER_OBJECT:
 			case GC_ObjectModel::SCAN_REFERENCE_MIXED_OBJECT:

diff --git a/runtime/gc_glue_java/CompactSchemeFixupObject.cpp b/runtime/gc_glue_java/CompactSchemeFixupObject.cpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2021 IBM Corp. and others
+ * Copyright (c) 1991, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -19,6 +19,7 @@
  *
  * SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
  *******************************************************************************/
+#include "j9.h"
 
 #include "CompactSchemeFixupObject.hpp"
 
@@ -31,6 +32,7 @@
 #include "MixedObjectIterator.hpp"
 #include "ObjectAccessBarrier.hpp"
 #include "OwnableSynchronizerObjectBuffer.hpp"
+#include "VMHelpers.hpp"
 #include "ParallelDispatcher.hpp"
 #include "PointerContiguousArrayIterator.hpp"
 #include "FlattenedContiguousArrayIterator.hpp"
@@ -47,6 +49,39 @@ MM_CompactSchemeFixupObject::fixupMixedObject(omrobjectptr_t objectPtr)
 	}
 }
 
+void
+MM_CompactSchemeFixupObject::doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slot)
+{
+	*slot = _compactScheme->getForwardingPtr(*slot);
+}
+
+/**
+ * @todo Provide function documentation
+ */
+void
+stackSlotIteratorForCompactScheme(J9JavaVM *javaVM, J9Object **slotPtr, void *localData, J9StackWalkState *walkState, const void *stackLocation)
+{
+	StackIteratorData4CompactSchemeFixupObject *data = (StackIteratorData4CompactSchemeFixupObject *)localData;
+	data->compactSchemeFixupObject->doStackSlot(data->env, data->fromObject, slotPtr);
+}
+
+
+void
+MM_CompactSchemeFixupObject::fixupContinuationObject(MM_EnvironmentStandard *env, omrobjectptr_t objectPtr)
+{
+	fixupMixedObject(objectPtr);
+	/* fixup Java Stacks in J9VMContinuation */
+	J9VMThread *currentThread = (J9VMThread *)env->getLanguageVMThread();
+	if (VM_VMHelpers::needScanStacksForContinuation(currentThread, objectPtr)) {
+		StackIteratorData4CompactSchemeFixupObject localData;
+		localData.compactSchemeFixupObject = this;
+		localData.env = env;
+		localData.fromObject = objectPtr;
+
+		GC_VMThreadStackSlotIterator::scanSlots(currentThread, objectPtr, (void *)&localData, stackSlotIteratorForCompactScheme, false, false);
+	}
+}
+
 void
 MM_CompactSchemeFixupObject::fixupArrayObject(omrobjectptr_t objectPtr)
 {
@@ -104,7 +139,9 @@ MM_CompactSchemeFixupObject::fixupObject(MM_EnvironmentStandard *env, omrobjectp
 		addOwnableSynchronizerObjectInList(env, objectPtr);
 		fixupMixedObject(objectPtr);
 		break;
-
+	case GC_ObjectModel::SCAN_CONTINUATION_OBJECT:
+		fixupContinuationObject(env, objectPtr);
+		break;
 	case GC_ObjectModel::SCAN_FLATTENED_ARRAY_OBJECT:
 		fixupFlattenedArrayObject(objectPtr);
 		break;

diff --git a/runtime/gc_glue_java/CompactSchemeFixupObject.hpp b/runtime/gc_glue_java/CompactSchemeFixupObject.hpp
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2021 IBM Corp. and others
+ * Copyright (c) 1991, 2022 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -38,6 +38,7 @@ class MM_CompactSchemeFixupObject {
 	MM_CompactScheme *_compactScheme;
 public:
 
+	void doStackSlot(MM_EnvironmentBase *env, omrobjectptr_t fromObject, omrobjectptr_t *slot);
 	/**
 	 * Perform fixup for a single object
 	 * @param env[in] the current thread
@@ -73,6 +74,7 @@ class MM_CompactSchemeFixupObject {
 	 */
 	void fixupFlattenedArrayObject(omrobjectptr_t objectPtr);
 
+	void fixupContinuationObject(MM_EnvironmentStandard *env, omrobjectptr_t objectPtr);
 	/**
 	 * Called whenever a ownable synchronizer object is fixed up during compact. Places the object on the thread-specific buffer of gc work thread.
 	 * @param env -- current thread environment
@@ -81,4 +83,10 @@ class MM_CompactSchemeFixupObject {
 	MMINLINE void addOwnableSynchronizerObjectInList(MM_EnvironmentBase *env, omrobjectptr_t objectPtr);
 };
 
+typedef struct StackIteratorData4CompactSchemeFixupObject {
+	MM_CompactSchemeFixupObject *compactSchemeFixupObject;
+	MM_EnvironmentBase *env;
+	J9Object *fromObject;
+} StackIteratorData4CompactSchemeFixupObject;
+
 #endif /* COMPACTSCHEMEOBJECTFIXUP_HPP_ */