Re-connect crash recursion protection with VM stackwalker

ddprof-lib/build.gradle

@@ -143,6 +143,9 @@ def copyUpstreamFiles = tasks.register('copyUpstreamFiles', Copy) {
   configure {
     dependsOn cloneAPTask
   }
+  onlyIf {
+    !project.hasProperty("debug-ap")
+  }
   description = 'Copy shared upstream files'
   from("${projectDir}/build/async-profiler/src") {
     include "arch.h"

ddprof-lib/src/main/cpp/profiler.cpp

@@ -910,7 +910,7 @@ bool Profiler::crashHandler(int signo, siginfo_t *siginfo, void *ucontext) {
   if (VM::isHotspot()) {
     // the following checks require vmstructs and therefore HotSpot
 
-    StackWalker::checkFault();
+    ddprof::StackWalker::checkFault(thrd);
 
     // Workaround for JDK-8313796. Setting cstack=dwarf also helps
     if (VMStructs::isInterpretedFrameValidFunc((const void *)pc) &&

ddprof-lib/src/main/cpp/stackWalker_dd.h

@@ -18,9 +18,19 @@
 #define _STACKWALKER_DD_H
 
 #include "stackWalker.h"
+#include "thread.h"
+#include "vmStructs_dd.h"
 
 
 namespace ddprof {
+  // === copied over from the upstream stackWalker.cpp because of visibility issues ===
+  const uintptr_t SAME_STACK_DISTANCE = 8192;
+
+  static inline bool sameStack(void* hi, void* lo) {
+    return (uintptr_t)hi - (uintptr_t)lo < SAME_STACK_DISTANCE;
+  }
+  // === end of copied code ===
+
   class StackWalker : public ::StackWalker {
     public:
       inline static int walkFP(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated) {
@@ -66,6 +76,20 @@ namespace ddprof {
         }
         return walked;
       }
+
+      static void checkFault(ProfiledThread* thrd) {
+        // We need to copy the following checks from the upstream stackWalker.cpp because we need a callback
+        // when the crash is actually handled and this is DD specific
+        VMThread *vm_thread = VMThread::current();
+        if (vm_thread != NULL && sameStack(vm_thread->exception(), &vm_thread)) {
+          if (thrd) {
+            // going to longjmp out of the signal handler, reset the crash handler depth counter
+            thrd->resetCrashHandler();
+          }
+        }
+        // delegate back to the upstream code
+        ::StackWalker::checkFault();
+      }
   };
 }