SNAP-2503: jvmkill issues SIGTERM before SIGKILL

gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/control/HeapMemoryMonitor.java

@@ -21,7 +21,9 @@
 import java.lang.management.MemoryPoolMXBean;
 import java.lang.management.MemoryType;
 import java.lang.management.MemoryUsage;
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Executors;
@@ -93,9 +95,6 @@ public final class HeapMemoryMonitor implements NotificationListener,
   private static final int POLLER_INTERVAL =
        Integer.getInteger(POLLER_INTERVAL_PROP, 500);
 
-  // Duration in millis to wait for jmap -histo to finish
-  private static final int JMAP_HISTO_SLEEP_DURATION = 3 * 1000;
-
   // This holds a new event as it transitions from updateStateAndSendEvent(...) to fillInProfile()
   private ThreadLocal<MemoryEvent> upcomingEvent = new ThreadLocal<MemoryEvent>();
 
@@ -693,17 +692,34 @@ public void jmapDump(String pid) {
       SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss");
       String dateSuffix = dateFormat.format(new java.util.Date(
           System.currentTimeMillis()));
+      Integer waitTime;
+
       if (inputArgs.contains("-XX:+HeapDumpOnOutOfMemoryError")) {
+        Integer heapDumpTimeout = 30;
+        try {
+          Object[] filtArgs = inputArgs.stream()
+              .filter(e -> e.contains("snappydata.onCriticalHeapDumpTimeoutSeconds="))
+              .map(e -> e.split("=")[1]).toArray();
+          if (filtArgs.length > 0) {
+            heapDumpTimeout = Integer.parseInt(filtArgs[0].toString());
+          }
+        } catch (NumberFormatException e) {
+          logger.warn("Failed to parse user provided value for " +
+              "OnCriticalHeapDumpTimeoutSeconds. default value of 30 will be used");
+        }
+
         jmapCommand = new String[] { "/bin/sh", "-c", "jmap -dump:format=b,file=" +
             "java_pid" + pid + "-" + dateSuffix + ".hprof " + pid
         };
+        waitTime = heapDumpTimeout * 1000;
       } else {
         jmapCommand = new String[] { "/bin/sh", "-c", "jmap -histo " + pid + " > " +
             "java_pid" + pid + "-" + dateSuffix + ".jmap"
         };
+        waitTime = 3 * 1000;
       }
       Process jmapProcess = Runtime.getRuntime().exec(jmapCommand);
-      jmapProcess.waitFor(JMAP_HISTO_SLEEP_DURATION, TimeUnit.MILLISECONDS);
+      jmapProcess.waitFor(waitTime, TimeUnit.MILLISECONDS);
     } catch (Exception e) {
       logger.error("Failed to log heap histogram for pid: " + pid, e);
     }

gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/jvmkill.c

@@ -45,7 +45,23 @@ resourceExhausted(
       jint flags,
       const void *reserved,
       const char *description) {
-   logMessage("ResourceExhausted: %s: killing current process!", description);
+   int sleepSeconds = 0;
+   if (getenv("JVMKILL_SLEEP_SECONDS")) {
+     const char* s = getenv("JVMKILL_SLEEP_SECONDS");
+     char *ptr;
+     sleepSeconds = strtol(s, &ptr, 10);
+   }
+   if (sleepSeconds < 1) {
+     sleepSeconds = 30;
+   }
+
+   logMessage("ResourceExhausted: %s: terminating current process!\n", description);
+   kill(getpid(), SIGTERM);
+
+   logMessage("SIGKILL will be issued after %d seconds if pid doesn't exit\n", sleepSeconds);
+   sleep(sleepSeconds);
+
+   logMessage("ResourceExhausted: %s: killing current process!\n", description);
    kill(getpid(), SIGKILL);
 }
 
@@ -78,4 +94,4 @@ Agent_OnLoad(JavaVM *vm, char *options, void *reserved) {
    }
 
    return JNI_OK;
-}
+}