[NFC][ASAN] Replace AsanInitIsRunning with TryAsanInitFromRtl

Reviewers: zacklj89

Reviewed By: zacklj89

Pull Request: #74171

Author: vitalybuka

compiler-rt/lib/asan/asan_interceptors.cpp

@@ -96,14 +96,16 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
   ASAN_WRITE_RANGE(ctx, ptr, size)
 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
   ASAN_READ_RANGE(ctx, ptr, size)
-#  define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)    \
-    ASAN_INTERCEPTOR_ENTER(ctx, func);                \
-    do {                                              \
-      if (AsanInitIsRunning())                        \
-        return REAL(func)(__VA_ARGS__);               \
-      if (SANITIZER_APPLE && UNLIKELY(!AsanInited())) \
-        return REAL(func)(__VA_ARGS__);               \
-      ENSURE_ASAN_INITED();                           \
+#  define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
+    ASAN_INTERCEPTOR_ENTER(ctx, func);             \
+    do {                                           \
+      if constexpr (SANITIZER_APPLE) {             \
+        if (UNLIKELY(!AsanInited()))               \
+          return REAL(func)(__VA_ARGS__);          \
+      } else {                                     \
+        if (!TryAsanInitFromRtl())                 \
+          return REAL(func)(__VA_ARGS__);          \
+      }                                            \
     } while (false)
 #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
   do {                                            \
@@ -534,16 +536,16 @@ INTERCEPTOR(char*, strncat, char *to, const char *from, uptr size) {
 INTERCEPTOR(char *, strcpy, char *to, const char *from) {
   void *ctx;
   ASAN_INTERCEPTOR_ENTER(ctx, strcpy);
-#if SANITIZER_APPLE
-  if (UNLIKELY(!AsanInited()))
-    return REAL(strcpy)(to, from);
-#endif
-  // strcpy is called from malloc_default_purgeable_zone()
-  // in __asan::ReplaceSystemAlloc() on Mac.
-  if (AsanInitIsRunning()) {
-    return REAL(strcpy)(to, from);
+  if constexpr (SANITIZER_APPLE) {
+    // strcpy is called from malloc_default_purgeable_zone()
+    // in __asan::ReplaceSystemAlloc() on Mac.
+    if (UNLIKELY(!AsanInited()))
+      return REAL(strcpy)(to, from);
+  } else {
+    if (!TryAsanInitFromRtl())
+      return REAL(strcpy)(to, from);
   }
-  ENSURE_ASAN_INITED();
+
   if (flags()->replace_str) {
     uptr from_size = internal_strlen(from) + 1;
     CHECK_RANGES_OVERLAP("strcpy", to, from_size, from, from_size);

compiler-rt/lib/asan/asan_internal.h

@@ -60,6 +60,7 @@ class AsanThread;
 using __sanitizer::StackTrace;
 
 void AsanInitFromRtl();
+bool TryAsanInitFromRtl();
 
 // asan_win.cpp
 void InitializePlatformExceptionHandlers();
@@ -131,7 +132,6 @@ void InstallAtExitCheckLeaks();
   __asan_on_error()
 
 bool AsanInited();
-bool AsanInitIsRunning();  // Used to avoid infinite recursion in __asan_init().
 extern bool replace_intrin_cached;
 extern void (*death_callback)(void);
 // These magic values are written to shadow for better error

compiler-rt/lib/asan/asan_malloc_linux.cpp

@@ -31,7 +31,7 @@
 using namespace __asan;
 
 struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
-  static bool UseImpl() { return AsanInitIsRunning(); }
+  static bool UseImpl() { return !TryAsanInitFromRtl(); }
   static void OnAllocate(const void *ptr, uptr size) {
 #  if CAN_SANITIZE_LEAKS
     // Suppress leaks from dlerror(). Previously dlsym hack on global array was
@@ -65,23 +65,20 @@ INTERCEPTOR(void, cfree, void *ptr) {
 INTERCEPTOR(void*, malloc, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Allocate(size);
-  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_malloc(size, &stack);
 }
 
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Callocate(nmemb, size);
-  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_calloc(nmemb, size, &stack);
 }
 
 INTERCEPTOR(void*, realloc, void *ptr, uptr size) {
   if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
     return DlsymAlloc::Realloc(ptr, size);
-  ENSURE_ASAN_INITED();
   GET_STACK_TRACE_MALLOC;
   return asan_realloc(ptr, size, &stack);
 }

compiler-rt/lib/asan/asan_rtl.cpp

@@ -80,7 +80,7 @@ static void SetAsanInitIsRunning(u32 val) { asan_init_is_running = val; }
 
 bool AsanInited() { return asan_inited == 1; }
 
-bool AsanInitIsRunning() { return asan_init_is_running == 1; }
+static bool AsanInitIsRunning() { return asan_init_is_running == 1; }
 
 bool replace_intrin_cached;
 
@@ -525,6 +525,14 @@ void AsanInitFromRtl() {
     AsanInitInternal();
 }
 
+bool TryAsanInitFromRtl() {
+  if (UNLIKELY(AsanInitIsRunning()))
+    return false;
+  if (UNLIKELY(!AsanInited()))
+    AsanInitInternal();
+  return true;
+}
+
 #if ASAN_DYNAMIC
 // Initialize runtime in case it's LD_PRELOAD-ed into unsanitized executable
 // (and thus normal initializers from .preinit_array or modules haven't run).