Add support for -XX:+HeapDumpOnOutOfMemoryError

docs/reference-manual/native-image/guides/create-heap-dump-from-native-executable.md

@@ -9,12 +9,13 @@ permalink: /reference-manual/native-image/guides/create-heap-dump/
 
 You can create a heap dump of a running executable to monitor its execution. Just like any other Java heap dump, it can be opened with the [VisualVM](../../../tools/visualvm.md) tool.
 
-To enable heap dump support, native executables must be built with the `--enable-monitoring=heapdump` option. Heap dumps can then be created in three different ways:
+To enable heap dump support, native executables must be built with the `--enable-monitoring=heapdump` option. Heap dumps can then be created in different ways:
 
 1. Create heap dumps with VisualVM.
 2. Dump the initial heap of a native executable using the `-XX:+DumpHeapAndExit` command-line option.
 3. Create heap dumps sending a `SIGUSR1` signal at run time.
 4. Create heap dumps programmatically using the [`org.graalvm.nativeimage.VMRuntime#dumpHeap`](https://github.com/oracle/graal/blob/master/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/VMInspectionOptions.java) API.
+5. If `-XX:+HeapDumpOnOutOfMemoryError` is passed in at runtime and an `OutOfMemoryError` is encountered, a heap dump will be produced.
 
 All approaches are described below.
 
@@ -275,6 +276,18 @@ The condition to create a heap dump is provided as an option on the command line
 
     ![Native Image Heap Dump View in VisualVM](img/heap-dump-api.png)
 
+## Create a Heap Dump on `OutOfMemoryError`
+
+Start the application with `-XX:+HeapDumpOnOutOfMemoryError` option to get a heap dump when an `OutOfMemoryError` occurs.
+The generated heap dump file name will have the `svm-heapdump-<PID>.hprof` naming format.
+For example:
+
+```shell
+./example -XX:+HeapDumpOnOutOfMemoryError
+Dumping heap to svm-heapdump-67799.hprof ...
+Exception in thread "main" java.lang.OutOfMemoryError: Garbage-collected heap size exceeded.
+```
+
 ### Related Documentation
 
 * [Debugging and Diagnostics](../DebuggingAndDiagnostics.md)

sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/VMRuntime.java

@@ -92,6 +92,20 @@ public static void dumpHeap(String outputFile, boolean live) throws IOException
         ImageSingletons.lookup(HeapDumpSupport.class).dumpHeap(outputFile, live);
     }
 
+    /**
+     * Dumps the heap to a predetermined file in case of an @{@link OutOfMemoryError}, in the same format as the hprof heap dump.
+     *
+     * @throws UnsupportedOperationException if this operation is not supported.
+     *
+     * @since 23.1
+     */
+    public static void dumpHeapOnOutOfMemoryError() {
+        if (!ImageSingletons.contains(HeapDumpSupport.class)) {
+            throw new UnsupportedOperationException();
+        }
+        ImageSingletons.lookup(HeapDumpSupport.class).dumpHeapOnOutOfMemoryError();
+    }
+
     private VMRuntime() {
     }
 }

sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/impl/HeapDumpSupport.java

@@ -44,4 +44,7 @@ public interface HeapDumpSupport {
 
     void dumpHeap(String outputFile, boolean live) throws java.io.IOException;
 
+    void dumpHeapOnOutOfMemoryError();
+
+    void initHeapDumpOnOutOfMemoryErrorPath();
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -29,12 +29,14 @@
 
 import java.lang.ref.Reference;
 
+import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicBoolean;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.IsolateThread;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.StackValue;
+import org.graalvm.nativeimage.VMRuntime;
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.nativeimage.c.struct.RawField;
 import org.graalvm.nativeimage.c.struct.RawStructure;
@@ -116,6 +118,8 @@ public final class GCImpl implements GC {
     private UnsignedWord collectionEpoch = WordFactory.zero();
     private long lastWholeHeapExaminedTimeMillis = -1;
 
+    private final AtomicBoolean outOfMemoryReported = new AtomicBoolean(false);
+
     @Platforms(Platform.HOSTED_ONLY.class)
     GCImpl() {
         this.policy = CollectionPolicy.getInitialPolicy();
@@ -150,10 +154,17 @@ public void maybeCollectOnAllocation() {
             outOfMemory = collectWithoutAllocating(GenScavengeGCCause.OnAllocation, false);
         }
         if (outOfMemory) {
+            reportJavaOutOfMemory();
             throw OutOfMemoryUtil.heapSizeExceeded();
         }
     }
 
+    private void reportJavaOutOfMemory() {
+        if (SubstrateOptions.isHeapDumpOnOutOfMemoryError() && outOfMemoryReported.compareAndSet(false, true)) {
+            VMRuntime.dumpHeapOnOutOfMemoryError();
+        }
+    }
+
     @Override
     public void maybeCauseUserRequestedCollection(GCCause cause, boolean fullGC) {
         if (policy.shouldCollectOnRequest(cause, fullGC)) {
@@ -165,6 +176,7 @@ private void collect(GCCause cause, boolean forceFullGC) {
         if (!hasNeverCollectPolicy()) {
             boolean outOfMemory = collectWithoutAllocating(cause, forceFullGC);
             if (outOfMemory) {
+                reportJavaOutOfMemory();
                 throw OutOfMemoryUtil.heapSizeExceeded();
             }
         }

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixRawFileOperationSupport.java

@@ -30,6 +30,7 @@
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
+import org.graalvm.nativeimage.c.type.CCharPointer;
 import org.graalvm.nativeimage.c.type.CTypeConversion;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.SignedWord;
@@ -60,13 +61,30 @@ public RawFileDescriptor create(File file, FileCreationMode creationMode, FileAc
         return open0(path, flags);
     }
 
+    @Override
+    public RawFileDescriptor create(CCharPointer cPath, FileCreationMode creationMode, FileAccessMode accessMode) {
+        int flags = parseMode(creationMode) | parseMode(accessMode);
+        return open0(cPath, flags);
+    }
+
+    private static RawFileDescriptor open0(CCharPointer cPath, int flags) {
+        int permissions = PosixStat.S_IRUSR() | PosixStat.S_IWUSR();
+        return WordFactory.signed(Fcntl.NoTransitions.open(cPath, flags, permissions));
+    }
+
     @Override
     public RawFileDescriptor open(File file, FileAccessMode mode) {
         String path = file.getPath();
         int flags = parseMode(mode);
         return open0(path, flags);
     }
 
+    @Override
+    public RawFileDescriptor open(CCharPointer cPath, FileAccessMode mode) {
+        int flags = parseMode(mode);
+        return open0(cPath, flags);
+    }
+
     private static RawFileDescriptor open0(String path, int flags) {
         int permissions = PosixStat.S_IRUSR() | PosixStat.S_IWUSR();
         try (CTypeConversion.CCharPointerHolder cPath = CTypeConversion.toCString(path)) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/DumpHeapOnSignalFeature.java

@@ -30,6 +30,7 @@
 import java.util.Date;
 import java.util.TimeZone;
 
+import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.ProcessProperties;
 import org.graalvm.nativeimage.VMRuntime;
 
@@ -39,6 +40,7 @@
 import com.oracle.svm.core.log.Log;
 
 import jdk.internal.misc.Signal;
+import org.graalvm.nativeimage.impl.HeapDumpSupport;
 
 @AutomaticallyRegisteredFeature
 public class DumpHeapOnSignalFeature implements InternalFeature {
@@ -59,6 +61,9 @@ final class DumpHeapStartupHook implements RuntimeSupport.Hook {
     public void execute(boolean isFirstIsolate) {
         if (isFirstIsolate && SubstrateOptions.EnableSignalHandling.getValue()) {
             DumpHeapReport.install();
+            if (SubstrateOptions.isHeapDumpOnOutOfMemoryError()) {
+                ImageSingletons.lookup(HeapDumpSupport.class).initHeapDumpOnOutOfMemoryErrorPath();
+            }
         }
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java

@@ -825,6 +825,13 @@ public Boolean getValue(OptionValues values) {
         }
     };
 
+    @Option(help = "Dump heap to file when java.lang.OutOfMemoryError is thrown.")//
+    public static final RuntimeOptionKey<Boolean> HeapDumpOnOutOfMemoryError = new RuntimeOptionKey<>(false, Immutable);
+
+    public static boolean isHeapDumpOnOutOfMemoryError() {
+        return VMInspectionOptions.hasHeapDumpSupport() && SubstrateOptions.HeapDumpOnOutOfMemoryError.getValue();
+    }
+
     @Option(help = "The path (filename or directory) where heap dumps are created (defaults to the working directory).")//
     public static final RuntimeOptionKey<String> HeapDumpPath = new RuntimeOptionKey<>("", Immutable);
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/dump/HeapDumpSupportImpl.java

@@ -25,19 +25,26 @@
 package com.oracle.svm.core.heap.dump;
 
 import static com.oracle.svm.core.heap.RestrictHeapAccess.Access.NO_ALLOCATION;
-import static com.oracle.svm.core.heap.RestrictHeapAccess.Access.UNRESTRICTED;
 
 import java.io.IOException;
 
+import com.oracle.svm.core.SubstrateOptions;
+import com.oracle.svm.core.SubstrateUtil;
+import com.oracle.svm.core.headers.LibC;
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
+import org.graalvm.nativeimage.ProcessProperties;
 import org.graalvm.nativeimage.StackValue;
+import org.graalvm.nativeimage.UnmanagedMemory;
 import org.graalvm.nativeimage.c.struct.RawField;
 import org.graalvm.nativeimage.c.struct.RawStructure;
 import org.graalvm.nativeimage.c.struct.SizeOf;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CTypeConversion;
 import org.graalvm.nativeimage.impl.HeapDumpSupport;
 import org.graalvm.word.Pointer;
+import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.UnmanagedMemoryUtil;
@@ -56,6 +63,7 @@
 public class HeapDumpSupportImpl implements HeapDumpSupport {
     private final HeapDumpWriter writer;
     private final HeapDumpOperation heapDumpOperation;
+    private CCharPointer heapOnErrorDumpPath;
 
     @Platforms(Platform.HOSTED_ONLY.class)
     public HeapDumpSupportImpl(HeapDumpMetadata metadata) {
@@ -77,6 +85,44 @@ public void dumpHeap(String filename, boolean gcBefore) throws IOException {
         }
     }
 
+    @Override
+    @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Heap dumping on OutOfMemoryError must not allocate.")
+    public void dumpHeapOnOutOfMemoryError() {
+        final Log log = Log.log();
+
+        final RawFileDescriptor fd = getFileSupport().create(heapOnErrorDumpPath, FileCreationMode.CREATE_OR_REPLACE, RawFileOperationSupport.FileAccessMode.READ_WRITE);
+        if (!getFileSupport().isValid(fd)) {
+            log.string("Invalid file descriptor opening heap dump on OutOfMemoryError.").newline();
+            return;
+        }
+
+        int size = SizeOf.get(HeapDumpVMOperationData.class);
+        HeapDumpVMOperationData data = StackValue.get(size);
+        UnmanagedMemoryUtil.fill((Pointer) data, WordFactory.unsigned(size), (byte) 0);
+
+        log.string("Dumping heap to ").string(heapOnErrorDumpPath).string(" ...").newline();
+
+        data.setGCBefore(false);
+        data.setRawFileDescriptor(fd);
+        heapDumpOperation.enqueue(data);
+    }
+
+    @Override
+    public void initHeapDumpOnOutOfMemoryErrorPath() {
+        String dumpFileName = "svm-heapdump-" + ProcessProperties.getProcessID() + ".hprof";
+        String dumpPath = SubstrateOptions.getHeapDumpPath(dumpFileName);
+        try (CTypeConversion.CCharPointerHolder cPath = CTypeConversion.toCString(dumpPath)) {
+            heapOnErrorDumpPath = copyDumpPath(cPath.get());
+        }
+    }
+
+    private static CCharPointer copyDumpPath(CCharPointer cPath) {
+        final UnsignedWord length = SubstrateUtil.strlen(cPath);
+        final CCharPointer copy = UnmanagedMemory.malloc(length);
+        LibC.memcpy(copy, cPath, length);
+        return copy;
+    }
+
     public void writeHeapTo(RawFileDescriptor fd, boolean gcBefore) throws IOException {
         int size = SizeOf.get(HeapDumpVMOperationData.class);
         HeapDumpVMOperationData data = StackValue.get(size);
@@ -136,13 +182,13 @@ protected void operate(NativeVMOperationData d) {
                 data.setSuccess(success);
             } catch (Throwable e) {
                 reportError(e);
+                data.setSuccess(false);
             }
         }
 
-        @RestrictHeapAccess(access = UNRESTRICTED, reason = "Error reporting may allocate.")
+        @RestrictHeapAccess(access = NO_ALLOCATION, reason = "Error reporting must not allocate.")
         private static void reportError(Throwable e) {
-            Log.log().string("An exception occurred during heap dumping. The data in the heap dump file may be corrupt.").newline().string(e.getClass().getName()).string(": ")
-                            .string(e.getMessage());
+            Log.log().string("An exception occurred during heap dumping. The data in the heap dump file may be corrupt: ").string(e.getClass().getName());
         }
     }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heapdump/HeapDumpSupportImpl.java

@@ -36,4 +36,14 @@ public void dumpHeap(String outputFile, boolean live) throws java.io.IOException
             com.oracle.svm.core.heapdump.HeapDumpWriter.singleton().writeHeapTo(fileOutputStream, live);
         }
     }
+
+    @Override
+    public void dumpHeapOnOutOfMemoryError() {
+        // No-op
+    }
+
+    @Override
+    public void initHeapDumpOnOutOfMemoryErrorPath() {
+        // No-op
+    }
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/RawFileOperationSupport.java

@@ -28,6 +28,7 @@
 
 import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
+import org.graalvm.nativeimage.c.type.CCharPointer;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordBase;
@@ -88,6 +89,15 @@ static RawFileOperationSupport nativeByteOrder() {
      */
     RawFileDescriptor create(File file, FileCreationMode creationMode, FileAccessMode accessMode);
 
+    /**
+     * Creates a file with the specified {@link FileCreationMode creation} and {@link FileAccessMode
+     * access modes}.
+     *
+     * @return If the operation is successful, it returns the file descriptor. Otherwise, it returns
+     *         a value where {@link #isValid} will return false.
+     */
+    RawFileDescriptor create(CCharPointer path, FileCreationMode creationMode, FileAccessMode accessMode);
+
     /**
      * Opens a file with the specified {@link FileAccessMode access mode}.
      *
@@ -104,6 +114,14 @@ static RawFileOperationSupport nativeByteOrder() {
      */
     RawFileDescriptor open(File file, FileAccessMode accessMode);
 
+    /**
+     * Opens a file with the specified {@link FileAccessMode access mode}.
+     *
+     * @return If the operation is successful, it returns the file descriptor. Otherwise, it returns
+     *         a value where {@link #isValid} will return false.
+     */
+    RawFileDescriptor open(CCharPointer path, FileAccessMode accessMode);
+
     /**
      * Checks if a file descriptor is valid or if it represents an error value.
      *