Skip to content

Commit

Permalink
[GR-56816] Improve serial GC heap verification.
Browse files Browse the repository at this point in the history 
Other small GC-related cleanups and improvements.
  • Loading branch information
@christianhaeubl
christianhaeubl committed Sep 4, 2024
1 parent bd8874b commit 2db30db
Show file tree
Hide file tree
Showing 13 changed files with 130 additions and 65 deletions.
3 changes: 2 additions & 1 deletion ...com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) {
}

public static void reset(AlignedHeader chunk) {
HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getEndOffset(chunk));
assert HeapChunk.getEndOffset(chunk).rawValue() == SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue();
initialize(chunk, WordFactory.unsigned(SerialAndEpsilonGCOptions.AlignedHeapChunkSize.getValue()));
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
Expand Down
26 changes: 22 additions & 4 deletions ...atevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import com.oracle.svm.core.code.RuntimeCodeInfoAccess;
import com.oracle.svm.core.code.RuntimeCodeInfoMemory;
import com.oracle.svm.core.code.SimpleCodeInfoQueryResult;
import com.oracle.svm.core.deopt.DeoptimizationSlotPacking;
import com.oracle.svm.core.deopt.DeoptimizedFrame;
import com.oracle.svm.core.deopt.Deoptimizer;
import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader;
Expand All @@ -80,6 +81,7 @@
import com.oracle.svm.core.heap.RestrictHeapAccess;
import com.oracle.svm.core.heap.RuntimeCodeCacheCleaner;
import com.oracle.svm.core.heap.VMOperationInfos;
import com.oracle.svm.core.interpreter.InterpreterSupport;
import com.oracle.svm.core.jdk.RuntimeSupport;
import com.oracle.svm.core.jfr.JfrGCWhen;
import com.oracle.svm.core.jfr.JfrTicks;
Expand Down Expand Up @@ -227,9 +229,13 @@ private void collectOperation(CollectionVMOperationData data) {
GenScavengeMemoryPoolMXBeans.notifyBeforeCollection();
HeapImpl.getAccounting().notifyBeforeCollection();

verifyBeforeGC();

boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC());
data.setOutOfMemory(outOfMemory);

verifyAfterGC();

HeapImpl.getAccounting().notifyAfterCollection();
GenScavengeMemoryPoolMXBeans.notifyAfterCollection();

Expand Down Expand Up @@ -305,11 +311,7 @@ private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean co

Timer collectionTimer = timers.collection.open();
try {
if (!followsIncremental) { // we would have verified the heap after the incremental GC
verifyBeforeGC();
}
scavenge(!complete);
verifyAfterGC();
if (complete) {
lastWholeHeapExaminedTimeMillis = System.currentTimeMillis();
}
Expand All @@ -329,6 +331,10 @@ private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean co

private void verifyBeforeGC() {
if (SubstrateGCOptions.VerifyHeap.getValue() && SerialGCOptions.VerifyBeforeGC.getValue()) {
if (SubstrateGCOptions.VerboseGC.getValue()) {
printGCPrefixAndTime().string("Verifying Before GC ").newline();
}

Timer verifyBeforeTimer = timers.verifyBefore.open();
try {
boolean success = true;
Expand All @@ -343,11 +349,19 @@ private void verifyBeforeGC() {
} finally {
verifyBeforeTimer.close();
}

if (SubstrateGCOptions.VerboseGC.getValue()) {
printGCPrefixAndTime().string("Verifying Before GC ").rational(timers.verifyBefore.getMeasuredNanos(), TimeUtils.nanosPerMilli, 3).string("ms").newline();
}
}
}

private void verifyAfterGC() {
if (SubstrateGCOptions.VerifyHeap.getValue() && SerialGCOptions.VerifyAfterGC.getValue()) {
if (SubstrateGCOptions.VerboseGC.getValue()) {
printGCPrefixAndTime().string("Verifying After GC ").newline();
}

Timer verifyAfterTime = timers.verifyAfter.open();
try {
boolean success = true;
Expand All @@ -362,6 +376,10 @@ private void verifyAfterGC() {
} finally {
verifyAfterTime.close();
}

if (SubstrateGCOptions.VerboseGC.getValue()) {
printGCPrefixAndTime().string("Verifying After GC ").rational(timers.verifyAfter.getMeasuredNanos(), TimeUtils.nanosPerMilli, 3).string("ms").newline();
}
}
}

Expand Down
35 changes: 27 additions & 8 deletions ...src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ private static boolean verifyRememberedSets() {
* reasonable state. Now, we can verify the remembered sets without having to worry about
* basic heap consistency.
*/
if (!SubstrateOptions.useRememberedSet() || !SerialGCOptions.VerifyRememberedSet.getValue()) {
if (!SerialGCOptions.useRememberedSet()) {
return true;
}

Expand Down Expand Up @@ -210,6 +210,7 @@ private static boolean verifyChunkList(Space space, String kind, HeapChunk.Heade

private static boolean verifyAlignedChunks(Space space, AlignedHeader firstAlignedHeapChunk) {
boolean success = true;

AlignedHeader aChunk = firstAlignedHeapChunk;
while (aChunk.isNonNull()) {
if (space != aChunk.getSpace()) {
Expand All @@ -228,6 +229,7 @@ private static boolean verifyAlignedChunks(Space space, AlignedHeader firstAlign

private static boolean verifyUnalignedChunks(Space space, UnalignedHeader firstUnalignedHeapChunk) {
boolean success = true;

UnalignedHeader uChunk = firstUnalignedHeapChunk;
while (uChunk.isNonNull()) {
if (space != uChunk.getSpace()) {
Expand Down Expand Up @@ -264,6 +266,11 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH
return false;
}

if (!HeapImpl.getHeap().getObjectHeader().isEncodedObjectHeader(header)) {
Log.log().string("Object ").zhex(ptr).string(" does not have a valid hub: ").zhex(header).newline();
return false;
}

if (ObjectHeaderImpl.isForwardedHeader(header)) {
Log.log().string("Object ").zhex(ptr).string(" has a forwarded header: ").zhex(header).newline();
return false;
Expand Down Expand Up @@ -315,12 +322,6 @@ private static boolean verifyObject(Object obj, AlignedHeader aChunk, UnalignedH
}
}

DynamicHub hub = KnownIntrinsics.readHub(obj);
if (!HeapImpl.getHeapImpl().isInImageHeap(hub)) {
Log.log().string("Object ").zhex(ptr).string(" references a hub that is not in the image heap: ").zhex(Word.objectToUntrackedPointer(hub)).newline();
return false;
}

return verifyReferences(obj);
}

Expand Down Expand Up @@ -365,13 +366,31 @@ private static boolean verifyReference(Object parentObject, Pointer reference, P
return false;
}

if (!ObjectHeaderImpl.getObjectHeaderImpl().pointsToObjectHeader(referencedObject)) {
Word header = ObjectHeader.readHeaderFromPointer(referencedObject);
if (!ObjectHeaderImpl.getObjectHeaderImpl().isEncodedObjectHeader(header)) {
Log.log().string("Object reference at ").zhex(reference).string(" does not point to a Java object or the object header of the Java object is invalid: ").zhex(referencedObject)
.string(". ");
printParent(parentObject);
return false;
}

if (ObjectHeaderImpl.isAlignedHeader(header)) {
AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(referencedObject);
if (referencedObject.belowThan(AlignedHeapChunk.getObjectsStart(chunk)) || referencedObject.aboveOrEqual(HeapChunk.getTopPointer(chunk))) {
Log.log().string("Object reference ").zhex(reference).string(" points to ").zhex(referencedObject).string(", which is outside the usable part of the corresponding aligned chunk.");
printParent(parentObject);
return false;
}
} else {
assert ObjectHeaderImpl.isUnalignedHeader(header);
UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunkFromObjectPointer(referencedObject);
if (referencedObject != UnalignedHeapChunk.getObjectStart(chunk)) {
Log.log().string("Object reference ").zhex(reference).string(" points to ").zhex(referencedObject).string(", which is outside the usable part of the corresponding unaligned chunk.");
printParent(parentObject);
return false;
}
}

return true;
}

Expand Down
10 changes: 2 additions & 8 deletions ...rc/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,19 +173,13 @@ public boolean isInReadOnlyHugePartition(Pointer ptr) {
}

/**
* This method only returns the correct result for pointers that point to the the start of an
* This method only returns the correct result for pointers that point to the start of an
* object. This is sufficient for all our current use cases. This code must be as fast as
* possible as the GC uses it for every visited reference.
*/
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public boolean isInImageHeap(Pointer objectPointer) {
boolean result;
if (objectPointer.isNull()) {
result = false;
} else {
result = objectPointer.aboveOrEqual(Word.objectToUntrackedPointer(firstObject)) && objectPointer.belowOrEqual(Word.objectToUntrackedPointer(lastObject));
}
return result;
return objectPointer.aboveOrEqual(Word.objectToUntrackedPointer(firstObject)) && objectPointer.belowOrEqual(Word.objectToUntrackedPointer(lastObject));
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
Expand Down
3 changes: 2 additions & 1 deletion ...ratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,7 @@ private Object copyAlignedObject(Object originalObj) {
if (probability(SLOW_PATH_PROBABILITY, ObjectHeaderImpl.hasIdentityHashFromAddressInline(header))) {
addIdentityHashField = true;
copySize = LayoutEncoding.getSizeFromObjectInlineInGC(originalObj, true);
assert copySize.aboveOrEqual(originalSize);
}
}

Expand Down Expand Up @@ -431,7 +432,7 @@ private Object copyAlignedObject(Object originalObj) {
// If the object was promoted to the old gen, we need to take care of the remembered
// set bit and the first object table (even when promoting from old to old).
AlignedHeapChunk.AlignedHeader copyChunk = AlignedHeapChunk.getEnclosingChunk(copy);
RememberedSet.get().enableRememberedSetForObject(copyChunk, copy);
RememberedSet.get().enableRememberedSetForObject(copyChunk, copy, copySize);
}
return copy;
}
Expand Down
9 changes: 7 additions & 2 deletions ...racle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import com.oracle.svm.core.SubstrateGCOptions;
import com.oracle.svm.core.SubstrateOptions;
import com.oracle.svm.core.Uninterruptible;
import com.oracle.svm.core.config.ConfigurationValues;
import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader;
import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
import com.oracle.svm.core.genscavenge.graal.nodes.FormatArrayNode;
Expand All @@ -74,6 +75,7 @@
import com.oracle.svm.core.threadlocal.FastThreadLocalBytes;
import com.oracle.svm.core.threadlocal.FastThreadLocalFactory;
import com.oracle.svm.core.threadlocal.FastThreadLocalWord;
import com.oracle.svm.core.util.UnsignedUtils;
import com.oracle.svm.core.util.VMError;

/**
Expand Down Expand Up @@ -414,11 +416,14 @@ private static UnsignedWord availableTlabMemory(Descriptor allocator) {

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private static void guaranteeZeroed(Pointer memory, UnsignedWord size) {
VMError.guarantee(UnsignedUtils.isAMultiple(size, WordFactory.unsigned(ConfigurationValues.getTarget().wordSize)));

Pointer pos = memory;
Pointer end = memory.add(size);
while (pos.belowThan(end)) {
VMError.guarantee(pos.readByte(0) == 0);
pos = pos.add(1);
Word v = pos.readWord(0);
VMError.guarantee(v.equal(0));
pos = pos.add(ConfigurationValues.getTarget().wordSize);
}
}

Expand Down
21 changes: 15 additions & 6 deletions ...ore.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,14 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP

@AlwaysInline("GC performance")
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) {
public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) {
Pointer fotStart = getFirstObjectTableStart(chunk);
Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
Pointer startOffset = Word.objectToUntrackedPointer(obj).subtract(objectsStart);
Pointer endOffset = LayoutEncoding.getObjectEndInGC(obj).subtract(objectsStart);

Word objPtr = Word.objectToUntrackedPointer(obj);
UnsignedWord startOffset = objPtr.subtract(objectsStart);
UnsignedWord endOffset = startOffset.add(objSize);

FirstObjectTable.setTableForObject(fotStart, startOffset, endOffset);
ObjectHeaderImpl.setRememberedSetBit(obj);
}
Expand All @@ -108,8 +111,9 @@ public static void enableRememberedSet(AlignedHeader chunk) {
Pointer top = HeapChunk.getTopPointer(chunk);
while (offset.belowThan(top)) {
Object obj = offset.toObject();
enableRememberedSetForObject(chunk, obj);
offset = offset.add(LayoutEncoding.getSizeFromObjectInGC(obj));
UnsignedWord objSize = LayoutEncoding.getSizeFromObjectInGC(obj);
enableRememberedSetForObject(chunk, obj, objSize);
offset = offset.add(objSize);
}
}

Expand Down Expand Up @@ -204,7 +208,7 @@ private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end,

public static boolean verify(AlignedHeader chunk) {
boolean success = true;
success &= CardTable.verify(getCardTableStart(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk));
success &= CardTable.verify(getCardTableStart(chunk), getCardTableEnd(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk));
success &= FirstObjectTable.verify(getFirstObjectTableStart(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk));
return success;
}
Expand Down Expand Up @@ -278,6 +282,11 @@ private static Pointer getCardTableStart(Pointer chunk) {
return chunk.add(getCardTableStartOffset());
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private static Pointer getCardTableEnd(AlignedHeader chunk) {
return getCardTableStart(chunk).add(getCardTableSize());
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
private static Pointer getFirstObjectTableStart(AlignedHeader chunk) {
return getFirstObjectTableStart(HeapChunk.asPointer(chunk));
Expand Down
61 changes: 42 additions & 19 deletions ...com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.oracle.svm.core.UnmanagedMemoryUtil;
import com.oracle.svm.core.genscavenge.HeapChunk;
import com.oracle.svm.core.genscavenge.HeapImpl;
import com.oracle.svm.core.genscavenge.SerialGCOptions;
import com.oracle.svm.core.genscavenge.graal.BarrierSnippets;
import com.oracle.svm.core.heap.ObjectReferenceVisitor;
import com.oracle.svm.core.heap.ReferenceAccess;
Expand Down Expand Up @@ -142,28 +143,42 @@ public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) {
return CardTable.memoryOffsetToIndex(roundedMemory);
}

public static boolean verify(Pointer cardTableStart, Pointer objectsStart, Pointer objectsLimit) {
public static boolean verify(Pointer cardTableStart, Pointer cardTableEnd, Pointer objectsStart, Pointer objectsLimit) {
boolean success = true;
Pointer curPtr = objectsStart;
while (curPtr.belowThan(objectsLimit)) {
// As we only use imprecise card marking at the moment, only the card at the address of
// the object may be dirty.
Object obj = curPtr.toObject();
UnsignedWord cardTableIndex = memoryOffsetToIndex(curPtr.subtract(objectsStart));
if (isClean(cardTableStart, cardTableIndex)) {
CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart);
InteriorObjRefWalker.walkObject(obj, CARD_TABLE_VERIFICATION_VISITOR);
success &= CARD_TABLE_VERIFICATION_VISITOR.success;

DynamicHub hub = KnownIntrinsics.readHub(obj);
if (hub.isReferenceInstanceClass()) {
// The referent field of java.lang.Reference is excluded from the reference map,
// so we need to verify it separately.
Reference<?> ref = (Reference<?>) obj;
success &= verifyReferent(ref, cardTableStart, objectsStart);
if (SerialGCOptions.VerifyRememberedSet.getValue()) {
Pointer curPtr = objectsStart;
while (curPtr.belowThan(objectsLimit)) {
// As we only use imprecise card marking at the moment, only the card at the address
// of the object may be dirty.
Object obj = curPtr.toObject();
UnsignedWord cardTableIndex = memoryOffsetToIndex(curPtr.subtract(objectsStart));
if (isClean(cardTableStart, cardTableIndex)) {
CARD_TABLE_VERIFICATION_VISITOR.initialize(obj, cardTableStart, objectsStart);
InteriorObjRefWalker.walkObject(obj, CARD_TABLE_VERIFICATION_VISITOR);
success &= CARD_TABLE_VERIFICATION_VISITOR.success;
CARD_TABLE_VERIFICATION_VISITOR.reset();

DynamicHub hub = KnownIntrinsics.readHub(obj);
if (hub.isReferenceInstanceClass()) {
// The referent field of java.lang.Reference is excluded from the reference
// map, so we need to verify it separately.
Reference<?> ref = (Reference<?>) obj;
success &= verifyReferent(ref, cardTableStart, objectsStart);
}
}
curPtr = LayoutEncoding.getObjectEndInGC(obj);
}
} else {
/* Do a basic sanity check of the card table data. */
Pointer pos = cardTableStart;
while (pos.belowThan(cardTableEnd)) {
byte v = pos.readByte(0);
if (v != DIRTY_ENTRY && v != CLEAN_ENTRY) {
Log.log().string("Card at ").zhex(pos).string(" is neither dirty nor clean: ").zhex(v).newline();
return false;
}
pos = pos.add(1);
}
curPtr = LayoutEncoding.getObjectEndInGC(obj);
}
return success;
}
Expand Down Expand Up @@ -202,12 +217,20 @@ private static class CardTableVerificationVisitor implements ObjectReferenceVisi

@SuppressWarnings("hiding")
public void initialize(Object parentObject, Pointer cardTableStart, Pointer objectsStart) {
assert this.parentObject == null && this.cardTableStart.isNull() && this.objectsStart.isNull() && !this.success;
this.parentObject = parentObject;
this.cardTableStart = cardTableStart;
this.objectsStart = objectsStart;
this.success = true;
}

public void reset() {
this.parentObject = null;
this.cardTableStart = WordFactory.nullPointer();
this.objectsStart = WordFactory.nullPointer();
this.success = false;
}

@Override
@SuppressFBWarnings(value = {"NS_DANGEROUS_NON_SHORT_CIRCUIT"}, justification = "Non-short circuit logic is used on purpose here.")
public boolean visitObjectReference(Pointer reference, boolean compressed, Object holderObject) {
Expand Down
Loading

0 comments on commit 2db30db

Please sign in to comment.