[GR-33223] Add protected memory between heap base and image heap.

PullRequest: graal/9590

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java

@@ -33,20 +33,22 @@
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.AlignedChunk;
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.Chunk;
 import com.oracle.svm.core.genscavenge.ChunkedImageHeapAllocator.UnalignedChunk;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.image.ImageHeap;
 import com.oracle.svm.core.image.ImageHeapLayoutInfo;
 
 public class ChunkedImageHeapLayouter extends AbstractImageHeapLayouter<ChunkedImageHeapPartition> {
     private final ImageHeapInfo heapInfo;
     private final long startOffset;
-    private final boolean compressedNullPadding;
+    private final int nullRegionSize;
     private final long hugeObjectThreshold;
     private ChunkedImageHeapAllocator allocator;
 
-    public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, boolean compressedNullPadding) {
+    public ChunkedImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, int nullRegionSize) {
+        assert startOffset == 0 || startOffset >= Heap.getHeap().getImageHeapOffsetInAddressSpace() : "must be relative to the heap base";
         this.heapInfo = heapInfo;
         this.startOffset = startOffset;
-        this.compressedNullPadding = compressedNullPadding;
+        this.nullRegionSize = nullRegionSize;
         this.hugeObjectThreshold = HeapPolicy.getLargeArrayThreshold().rawValue();
     }
 
@@ -67,8 +69,8 @@ protected long getHugeObjectThreshold() {
 
     @Override
     protected ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) {
-        assert !compressedNullPadding || AlignedHeapChunk.getObjectsStartOffset().aboveThan(0) : "Expecting header to pad start so object offsets are strictly greater than 0";
-        allocator = new ChunkedImageHeapAllocator(imageHeap, startOffset);
+        long position = startOffset + nullRegionSize;
+        allocator = new ChunkedImageHeapAllocator(imageHeap, position);
         for (ChunkedImageHeapPartition partition : getPartitions()) {
             partition.layout(allocator);
         }
@@ -78,28 +80,43 @@ protected ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) {
     private ImageHeapLayoutInfo populateInfoObjects(int dynamicHubCount) {
         // Determine writable start boundary from chunks: a chunk that contains writable objects
         // must also have a writable card table
-        long writableBegin = getWritablePrimitive().getStartOffset();
+        long offsetOfFirstWritableAlignedChunk = getWritablePrimitive().getStartOffset();
         for (AlignedChunk chunk : allocator.getAlignedChunks()) {
-            if (chunk.isWritable() && chunk.getBegin() < writableBegin) {
-                assert writableBegin <= chunk.getEnd();
-                writableBegin = chunk.getBegin();
+            if (chunk.isWritable() && chunk.getBegin() < offsetOfFirstWritableAlignedChunk) {
+                assert offsetOfFirstWritableAlignedChunk <= chunk.getEnd();
+                offsetOfFirstWritableAlignedChunk = chunk.getBegin();
                 break; // (chunks are in ascending memory order)
             }
         }
-        long firstWritableUnalignedChunk = -1;
+        long offsetOfFirstWritableUnalignedChunk = -1;
         for (UnalignedChunk chunk : allocator.getUnalignedChunks()) {
             if (chunk.isWritable()) {
-                firstWritableUnalignedChunk = chunk.getBegin();
+                offsetOfFirstWritableUnalignedChunk = chunk.getBegin();
             }
             break;
         }
 
+        initializeHeapInfo(dynamicHubCount, offsetOfFirstWritableAlignedChunk, offsetOfFirstWritableUnalignedChunk);
+        return createLayoutInfo(startOffset, offsetOfFirstWritableAlignedChunk);
+    }
+
+    private void initializeHeapInfo(int dynamicHubCount, long offsetOfFirstWritableAlignedChunk, long offsetOfFirstWritableUnalignedChunk) {
+        long writableAligned = offsetOfFirstWritableAlignedChunk;
+        long writableUnaligned = offsetOfFirstWritableUnalignedChunk;
+
+        if (startOffset == 0) {
+            // Adjust all offsets by the offset of the image heap in the address space.
+            int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace();
+            writableAligned += imageHeapOffsetInAddressSpace;
+            if (writableUnaligned >= 0) {
+                writableUnaligned += imageHeapOffsetInAddressSpace;
+            }
+        }
+
         heapInfo.initialize(getReadOnlyPrimitive().firstObject, getReadOnlyPrimitive().lastObject, getReadOnlyReference().firstObject, getReadOnlyReference().lastObject,
                         getReadOnlyRelocatable().firstObject, getReadOnlyRelocatable().lastObject, getWritablePrimitive().firstObject, getWritablePrimitive().lastObject,
                         getWritableReference().firstObject, getWritableReference().lastObject, getWritableHuge().firstObject, getWritableHuge().lastObject,
-                        getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, writableBegin, firstWritableUnalignedChunk, dynamicHubCount);
-
-        return createLayoutInfo(startOffset, writableBegin);
+                        getReadOnlyHuge().firstObject, getReadOnlyHuge().lastObject, writableAligned, writableUnaligned, dynamicHubCount);
     }
 
     @Override

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java

@@ -29,6 +29,7 @@
 import java.util.List;
 
 import org.graalvm.compiler.api.replacements.Fold;
+import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.compiler.core.common.SuppressFBWarnings;
 import org.graalvm.compiler.nodes.gc.BarrierSet;
 import org.graalvm.compiler.word.Word;
@@ -87,6 +88,8 @@ public final class HeapImpl extends Heap {
     private static final VMMutex REF_MUTEX = new VMMutex("referencePendingList");
     private static final VMCondition REF_CONDITION = new VMCondition(REF_MUTEX);
 
+    private final int pageSize;
+
     // Singleton instances, created during image generation.
     private final YoungGeneration youngGeneration = new YoungGeneration("YoungGeneration");
     private final OldGeneration oldGeneration = new OldGeneration("OldGeneration");
@@ -111,7 +114,8 @@ public final class HeapImpl extends Heap {
     private List<Class<?>> classList;
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public HeapImpl(FeatureAccess access) {
+    public HeapImpl(FeatureAccess access, int pageSize) {
+        this.pageSize = pageSize;
         this.gcImpl = new GCImpl(access);
         this.runtimeCodeInfoGcSupport = new RuntimeCodeInfoGCSupportImpl();
         this.heapPolicy = new HeapPolicy();
@@ -421,15 +425,34 @@ public static boolean usesImageHeapCardMarking() {
     @Fold
     @Override
     public int getPreferredAddressSpaceAlignment() {
-        if (usesImageHeapChunks()) {
-            return UnsignedUtils.safeToInt(HeapPolicy.getAlignedHeapChunkAlignment());
-        }
-        return ConfigurationValues.getObjectLayout().getAlignment();
+        return UnsignedUtils.safeToInt(HeapPolicy.getAlignedHeapChunkAlignment());
     }
 
     @Fold
     @Override
     public int getImageHeapOffsetInAddressSpace() {
+        if (SubstrateOptions.SpawnIsolates.getValue() && SubstrateOptions.UseNullRegion.getValue() && CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment()) {
+            /*
+             * The image heap will be mapped in a way that there is a memory protected gap between
+             * the heap base and the start of the image heap. The gap won't need any memory in the
+             * native image file.
+             */
+            return NumUtil.safeToInt(HeapPolicyOptions.AlignedHeapChunkSize.getValue());
+        }
+        return 0;
+    }
+
+    @Fold
+    @Override
+    public int getImageHeapNullRegionSize() {
+        if (SubstrateOptions.SpawnIsolates.getValue() && SubstrateOptions.UseNullRegion.getValue() && !CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment()) {
+            /*
+             * Prepend a single null page to the image heap so that there is a memory protected gap
+             * between the heap base and the start of the image heap. The null page is placed
+             * directly into the native image file, so it makes the file slightly larger.
+             */
+            return pageSize;
+        }
         return 0;
     }
 
@@ -605,9 +628,15 @@ public Reference<?> getAndClearReferencePendingList() {
 
     @Override
     public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaHeapAccess) {
-        if (SubstrateOptions.SpawnIsolates.getValue() && value.equal(KnownIntrinsics.heapBase())) {
-            log.string("is the heap base");
-            return true;
+        if (SubstrateOptions.SpawnIsolates.getValue()) {
+            Pointer heapBase = KnownIntrinsics.heapBase();
+            if (value.equal(heapBase)) {
+                log.string("is the heap base");
+                return true;
+            } else if (value.aboveThan(heapBase) && value.belowThan(getImageHeapStart())) {
+                log.string("points into the protected memory between the heap base and the image heap");
+                return true;
+            }
         }
 
         Pointer ptr = (Pointer) value;
@@ -623,6 +652,16 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH
         return false;
     }
 
+    static Pointer getImageHeapStart() {
+        int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace();
+        if (imageHeapOffsetInAddressSpace > 0) {
+            return KnownIntrinsics.heapBase().add(imageHeapOffsetInAddressSpace);
+        } else {
+            int nullRegionSize = Heap.getHeap().getImageHeapNullRegionSize();
+            return KnownIntrinsics.heapBase().add(nullRegionSize);
+        }
+    }
+
     private boolean printLocationInfo(Log log, Pointer ptr) {
         if (imageHeapInfo.isInReadOnlyPrimitivePartition(ptr)) {
             log.string("points into the image heap (read-only primitives)");

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java

@@ -70,6 +70,7 @@ public final class ImageHeapInfo {
     @UnknownObjectField(types = Object.class) public Object firstObject;
     @UnknownObjectField(types = Object.class) public Object lastObject;
 
+    // All offsets are relative to the heap base.
     @UnknownPrimitiveField public long offsetOfFirstWritableAlignedChunk;
     @UnknownPrimitiveField public long offsetOfFirstWritableUnalignedChunk;
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java

@@ -25,14 +25,12 @@
 package com.oracle.svm.core.genscavenge;
 
 import org.graalvm.compiler.word.Word;
-import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
 
-import com.oracle.svm.core.Isolates;
 import com.oracle.svm.core.MemoryWalker;
 import com.oracle.svm.core.annotate.AlwaysInline;
 import com.oracle.svm.core.heap.ObjectVisitor;
@@ -94,7 +92,7 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject
         if (HeapImpl.usesImageHeapChunks()) {
             Pointer base = WordFactory.zero();
             if (!CommittedMemoryProvider.get().guaranteesHeapPreferredAddressSpaceAlignment()) {
-                base = (Pointer) Isolates.getHeapBase(CurrentIsolate.getIsolate());
+                base = HeapImpl.getImageHeapStart();
             }
             Pointer offset = current.subtract(base);
             UnsignedWord chunkOffset = alignedChunks ? UnsignedUtils.roundDown(offset, HeapPolicy.getAlignedHeapChunkAlignment())

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LinearImageHeapLayouter.java

@@ -25,18 +25,22 @@
 package com.oracle.svm.core.genscavenge;
 
 import com.oracle.svm.core.config.ConfigurationValues;
+import com.oracle.svm.core.heap.Heap;
 import com.oracle.svm.core.image.ImageHeap;
 import com.oracle.svm.core.image.ImageHeapLayoutInfo;
 
 public class LinearImageHeapLayouter extends AbstractImageHeapLayouter<LinearImageHeapPartition> {
     private final ImageHeapInfo heapInfo;
     private final long startOffset;
-    private final boolean compressedNullPadding;
+    private final int nullRegionSize;
 
-    public LinearImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, boolean compressedNullPadding) {
+    public LinearImageHeapLayouter(ImageHeapInfo heapInfo, long startOffset, int nullRegionSize) {
+        assert startOffset >= 0;
+        assert startOffset == 0 || startOffset >= Heap.getHeap().getImageHeapOffsetInAddressSpace() : "must be relative to the heap base";
+        assert nullRegionSize >= 0;
         this.heapInfo = heapInfo;
         this.startOffset = startOffset;
-        this.compressedNullPadding = compressedNullPadding;
+        this.nullRegionSize = nullRegionSize;
     }
 
     @Override
@@ -51,14 +55,8 @@ protected LinearImageHeapPartition createPartition(String name, boolean contains
 
     @Override
     protected ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) {
-        long beginOffset = startOffset;
-        if (compressedNullPadding) {
-            /*
-             * Zero designates null, so adding some explicit padding at the beginning of the native
-             * image heap is the easiest approach to make object offsets strictly greater than 0.
-             */
-            beginOffset += ConfigurationValues.getObjectLayout().getAlignment();
-        }
+        long beginOffset = startOffset + spaceReservedForNull();
+        assert beginOffset >= ConfigurationValues.getObjectLayout().getAlignment() : "Zero designates null";
         LinearImageHeapAllocator allocator = new LinearImageHeapAllocator(beginOffset);
         for (LinearImageHeapPartition partition : getPartitions()) {
             partition.allocateObjects(allocator);
@@ -67,6 +65,13 @@ protected ImageHeapLayoutInfo doLayout(ImageHeap imageHeap) {
         return createLayoutInfo(startOffset, getWritablePrimitive().getStartOffset());
     }
 
+    private long spaceReservedForNull() {
+        if (startOffset == 0 && nullRegionSize == 0) {
+            return ConfigurationValues.getObjectLayout().getAlignment();
+        }
+        return nullRegionSize;
+    }
+
     /**
      * Store which objects are at the boundaries of the image heap partitions. Here, we also merge
      * the read-only reference partition with the read-only relocatable partition.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/HeapFeature.java

@@ -74,7 +74,7 @@ public List<Class<? extends Feature>> getRequiredFeatures() {
 
     @Override
     public void afterRegistration(AfterRegistrationAccess access) {
-        HeapImpl heap = new HeapImpl(access);
+        HeapImpl heap = new HeapImpl(access, SubstrateOptions.getPageSize());
         ImageSingletons.add(Heap.class, heap);
         ImageSingletons.add(SubstrateAllocationSnippets.class, new GenScavengeAllocationSnippets());
         ImageSingletons.add(RememberedSet.class, createRememberedSet());
@@ -106,10 +106,11 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
     @Override
     public void afterAnalysis(AfterAnalysisAccess access) {
         ImageHeapLayouter heapLayouter;
+        int imageHeapNullRegionSize = Heap.getHeap().getImageHeapNullRegionSize();
         if (HeapImpl.usesImageHeapChunks()) { // needs CommittedMemoryProvider: registered late
-            heapLayouter = new ChunkedImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0, true);
+            heapLayouter = new ChunkedImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0, imageHeapNullRegionSize);
         } else {
-            heapLayouter = new LinearImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0, true);
+            heapLayouter = new LinearImageHeapLayouter(HeapImpl.getImageHeapInfo(), 0, imageHeapNullRegionSize);
         }
         ImageSingletons.add(ImageHeapLayouter.class, heapLayouter);
     }

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSegfaultHandler.java

@@ -85,7 +85,7 @@ protected void printSignalInfo(Log log, PointerBase signalInfo) {
             if (sigInfo.si_errno() != 0) {
                 log.string(", si_errno: ").signed(sigInfo.si_errno());
             }
-            log.string(", si_addr: ").signed(sigInfo.si_addr());
+            log.string(", si_addr: ").zhex(sigInfo.si_addr());
             log.newline();
         }
     }