Fix jemalloc crash on mac (#8847)

close #8843
pingcap · Mar 14, 2024 · d2a6642 · d2a6642
1 parent d1eac2b
commit d2a6642
Show file tree

Hide file tree

Showing 14 changed files with 70 additions and 58 deletions.
diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -112,6 +112,12 @@ elseif (OS_FREEBSD)
     set (JEMALLOC_INCLUDE_PREFIX "include_freebsd")
 elseif (APPLE)
     set (JEMALLOC_INCLUDE_PREFIX "include_darwin")
+
+    # Jemalloc is not supposed to replace `malloc`/`free`/etc. on MacOS.
+    # So we should explicitly prefix the jemalloc symbols. 
+    # See https://github.com/jemalloc/jemalloc/blob/1978e5cdac731dca43b62e4b03612c0758f7cece/INSTALL.md?plain=1#L92-L94
+    set (JEMALLOC_PREFIX "je_")
+    set (JEMALLOC_CPREFIX "JE_")
 else ()
     message (FATAL_ERROR "internal jemalloc: This OS is not supported")
 endif ()
@@ -134,10 +140,16 @@ endif ()
 
 configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in
         ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h)
+configure_file(include/jemalloc/jemalloc_rename.h.in include/jemalloc/jemalloc_rename.h)
+
 target_include_directories(jemalloc SYSTEM PRIVATE
         "${CMAKE_CURRENT_BINARY_DIR}/${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal")
 
-target_include_directories(jemalloc PUBLIC ${JEMALLOC_SOURCE_DIR}/include ${TiFlash_SOURCE_DIR}/contrib/jemalloc-cmake/include)
+target_include_directories(jemalloc PUBLIC
+        ${CMAKE_CURRENT_BINARY_DIR}/include
+        ${CMAKE_CURRENT_BINARY_DIR}/include/jemalloc
+        ${JEMALLOC_SOURCE_DIR}/include
+        ${TiFlash_SOURCE_DIR}/contrib/jemalloc-cmake/include)
 
 target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
 

diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h
@@ -10,9 +10,9 @@ extern "C" {
 #endif
 
 #include <jemalloc/jemalloc_defs.h>
+#include <jemalloc/jemalloc_rename.h>
 #include <jemalloc/jemalloc_macros.h>
 #include <jemalloc/jemalloc_protos.h>
-#include <jemalloc/jemalloc_rename.h>
 #include <jemalloc/jemalloc_typedefs.h>
 
 #if !defined(__clang__)

diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h
diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h.in b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h.in
@@ -0,0 +1,31 @@
+/*
+ * Name mangling for public symbols is controlled by --with-mangling and
+ * --with-jemalloc-prefix.  With default settings the je_ prefix is stripped by
+ * these macro definitions.
+ */
+#ifndef JEMALLOC_NO_RENAME
+#define je_aligned_alloc @JEMALLOC_PREFIX@aligned_alloc
+#define je_calloc @JEMALLOC_PREFIX@calloc
+#define je_dallocx @JEMALLOC_PREFIX@dallocx
+#define je_free @JEMALLOC_PREFIX@free
+#define je_mallctl @JEMALLOC_PREFIX@mallctl
+#define je_mallctlbymib @JEMALLOC_PREFIX@mallctlbymib
+#define je_mallctlnametomib @JEMALLOC_PREFIX@mallctlnametomib
+#define je_malloc @JEMALLOC_PREFIX@malloc
+#define je_malloc_conf @JEMALLOC_PREFIX@malloc_conf
+#define je_malloc_conf_2_conf_harder @JEMALLOC_PREFIX@malloc_conf_2_conf_harder
+#define je_malloc_message @JEMALLOC_PREFIX@malloc_message
+#define je_malloc_stats_print @JEMALLOC_PREFIX@malloc_stats_print
+#define je_malloc_usable_size @JEMALLOC_PREFIX@malloc_usable_size
+#define je_mallocx @JEMALLOC_PREFIX@mallocx
+#define je_smallocx_ca709c3139f77f4c00a903cdee46d71e9028f6c6 @JEMALLOC_PREFIX@smallocx_ca709c3139f77f4c00a903cdee46d71e9028f6c6
+#define je_nallocx @JEMALLOC_PREFIX@nallocx
+#define je_posix_memalign @JEMALLOC_PREFIX@posix_memalign
+#define je_rallocx @JEMALLOC_PREFIX@rallocx
+#define je_realloc @JEMALLOC_PREFIX@realloc
+#define je_sallocx @JEMALLOC_PREFIX@sallocx
+#define je_sdallocx @JEMALLOC_PREFIX@sdallocx
+#define je_xallocx @JEMALLOC_PREFIX@xallocx
+#define je_memalign @JEMALLOC_PREFIX@memalign
+#define je_valloc @JEMALLOC_PREFIX@valloc
+#endif
diff --git a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-#define JEMALLOC_PREFIX "je_"
-#define JEMALLOC_CPREFIX "JE_"
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-#define JEMALLOC_PREFIX "je_"
-#define JEMALLOC_CPREFIX "JE_"
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/...ib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/...ib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -6,8 +6,8 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "@JEMALLOC_PREFIX@"
+#define JEMALLOC_CPREFIX "@JEMALLOC_CPREFIX@"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are

diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp
@@ -358,13 +358,13 @@ void AsynchronousMetrics::update()
     M("background_thread.num_runs", uint64_t)  \
     M("background_thread.run_interval", uint64_t)
 
-#define GET_JEMALLOC_METRIC(NAME, TYPE)                    \
-    do                                                     \
-    {                                                      \
-        TYPE value{};                                      \
-        size_t size = sizeof(value);                       \
-        mallctl("stats." NAME, &value, &size, nullptr, 0); \
-        set("jemalloc." NAME, value);                      \
+#define GET_JEMALLOC_METRIC(NAME, TYPE)                       \
+    do                                                        \
+    {                                                         \
+        TYPE value{};                                         \
+        size_t size = sizeof(value);                          \
+        je_mallctl("stats." NAME, &value, &size, nullptr, 0); \
+        set("jemalloc." NAME, value);                         \
     } while (0);
 
         FOR_EACH_METRIC(GET_JEMALLOC_METRIC);