[rtsan] Warn if instrumented rtsan library opened via dlopen and inte…

…rceptors are not working (#119029)

compiler-rt/lib/rtsan/rtsan.cpp

@@ -83,6 +83,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init() {
 
   SanitizerToolName = "RealtimeSanitizer";
   InitializeFlags();
+
+  InitializePlatformEarly();
+
   InitializeInterceptors();
 
   InitializeSuppressions();

compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp

@@ -19,7 +19,10 @@ extern "C" const char *__rtsan_default_options() {
   // and make sure we do not overwhelm the syslog while testing. Also, let's
   // turn symbolization off to speed up testing, especially when not running
   // with llvm-symbolizer but with atos.
-  return "symbolize=false:abort_on_error=0:log_to_syslog=0";
+  return "symbolize=false:"
+         "abort_on_error=0:"
+         "log_to_syslog=0:"
+         "verify_interceptors=0:"; // some of our tests don't need interceptors
 #else
   // Let's turn symbolization off to speed up testing (more than 3 times speedup
   // observed).

compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp

@@ -972,8 +972,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
 LowLevelAllocator allocator_for_env;
 
 static bool ShouldCheckInterceptors() {
-  // Restrict "interceptors working?" check to ASan and TSan.
-  const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
+  // Restrict "interceptors working?" check
+  const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer",
+                                   "RealtimeSanitizer"};
   size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
   for (size_t i = 0; i < count; i++) {
     if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)

compiler-rt/test/rtsan/Darwin/dlopen.cpp

@@ -0,0 +1,44 @@
+// Checks that on OS X 10.11+ dlopen'ing a RTsanified library from a
+// non-instrumented program exits with a user-friendly message.
+
+// REQUIRES: osx-autointerception
+
+// XFAIL: ios
+
+// RUN: %clangxx -fsanitize=realtime %s -o %t.so -shared -DSHARED_LIB
+// RUN: %clangxx %s -o %t
+
+// RUN: RTSAN_DYLIB_PATH=`%clangxx -fsanitize=realtime %s -### 2>&1 \
+// RUN:   | grep "libclang_rt.rtsan_osx_dynamic.dylib" \
+// RUN:   | sed -e 's/.*"\(.*libclang_rt.rtsan_osx_dynamic.dylib\)".*/\1/'`
+
+// Launching a non-instrumented binary that dlopen's an instrumented library should fail.
+// RUN: not %run %t %t.so 2>&1 | FileCheck %s --check-prefix=CHECK-FAIL
+// Launching a non-instrumented binary with an explicit DYLD_INSERT_LIBRARIES should work.
+// RUN: DYLD_INSERT_LIBRARIES=$RTSAN_DYLIB_PATH %run %t %t.so 2>&1 | FileCheck %s
+
+// Launching an instrumented binary with the DYLD_INSERT_LIBRARIES env variable has no error
+// RUN: %clangxx -fsanitize=realtime %s -o %t
+// RUN: DYLD_INSERT_LIBRARIES=$RTSAN_DYLIB_PATH %run %t %t.so 2>&1 | FileCheck %s --check-prefix=CHECK-INSTRUMENTED
+
+#include <dlfcn.h>
+#include <stdio.h>
+
+#if defined(SHARED_LIB)
+extern "C" void foo() { fprintf(stderr, "Hello world.\n"); }
+#else  // defined(SHARED_LIB)
+int main(int argc, char *argv[]) {
+  void *handle = dlopen(argv[1], RTLD_NOW);
+  void (*foo)() = (void (*)())dlsym(handle, "foo");
+  foo();
+}
+#endif // defined(SHARED_LIB)
+
+// CHECK: Hello world.
+// CHECK-NOT: ERROR: Interceptors are not working.
+
+// CHECK-FAIL-NOT: Hello world.
+// CHECK-FAIL: ERROR: Interceptors are not working.
+
+// CHECK-INSTRUMENTED-NOT: ERROR: Interceptors are not working
+// CHECK-INSTRUMENTED: Hello world.