Merge pull request #941 from ldorau/Link_statically_with_custom_jemal…

…loc_with_disabled_initial_TLS

Link statically with custom jemalloc with disabled initial TLS

.github/docker/ubuntu-20.04.Dockerfile

@@ -24,7 +24,6 @@ ARG BASE_DEPS="\
 
 # UMF's dependencies
 ARG UMF_DEPS="\
-	libjemalloc-dev \
 	libhwloc-dev \
 	libtbb-dev"
 
@@ -34,6 +33,7 @@ ARG TEST_DEPS="\
 
 # Miscellaneous for our builds/CI (optional)
 ARG MISC_DEPS="\
+	automake \
 	clang \
 	g++-7 \
 	python3-pip \

.github/docker/ubuntu-22.04.Dockerfile

@@ -24,7 +24,6 @@ ARG BASE_DEPS="\
 
 # UMF's dependencies
 ARG UMF_DEPS="\
-	libjemalloc-dev \
 	libhwloc-dev \
 	libtbb-dev"
 
@@ -34,6 +33,7 @@ ARG TEST_DEPS="\
 
 # Miscellaneous for our builds/CI (optional)
 ARG MISC_DEPS="\
+	automake \
 	clang \
 	python3-pip \
 	sudo \

.github/workflows/coverity.yml

@@ -31,7 +31,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev
+        sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev
 
     - name: Download Coverity
       run: |

.github/workflows/nightly.yml

@@ -67,7 +67,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind
+        sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind
 
     - name: Configure CMake
       run: >

.github/workflows/reusable_basic.yml

@@ -124,7 +124,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang cmake libnuma-dev libjemalloc-dev lcov
+        sudo apt-get install -y clang cmake libnuma-dev lcov
 
     - name: Install TBB apt package
       if: matrix.install_tbb == 'ON'
@@ -209,7 +209,6 @@ jobs:
         --install-dir ${{env.INSTL_DIR}}
         --build-type ${{matrix.build_type}}
         --disjoint-pool
-        --jemalloc-pool
         ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.shared_library == 'ON' && '--proxy' || '' }}
         --umf-version ${{env.UMF_VERSION}}
         ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }}
@@ -300,7 +299,6 @@ jobs:
         --install-dir ${{env.INSTL_DIR}}
         --build-type ${{matrix.build_type}}
         --disjoint-pool
-        --jemalloc-pool
         ${{matrix.shared_library == 'ON' && '--proxy' || '' }}
         --umf-version ${{env.UMF_VERSION}}
         ${{ matrix.shared_library == 'ON' && '--shared-library' || ''}}
@@ -469,7 +467,7 @@ jobs:
         python3 -m pip install -r third_party/requirements.txt
 
     - name: Install hwloc
-      run: brew install hwloc jemalloc tbb
+      run: brew install hwloc tbb automake
 
     - name: Configure build
       run: >
@@ -495,7 +493,6 @@ jobs:
         --install-dir ${{env.INSTL_DIR}}
         --build-type ${{env.BUILD_TYPE}}
         --disjoint-pool
-        --jemalloc-pool
         --proxy
         --umf-version ${{env.UMF_VERSION}}
         --shared-library

.github/workflows/reusable_benchmarks.yml

@@ -34,7 +34,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
+          sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
 
       - name: Initialize vcpkg
         if: matrix.os == 'windows-latest'

.github/workflows/reusable_codeql.yml

@@ -62,7 +62,7 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
+        sudo apt-get install -y cmake clang libhwloc-dev libnuma-dev libtbb-dev
 
     # Latest distros do not allow global pip installation
     - name: "[Lin] Install Python requirements in venv"

.github/workflows/reusable_fast.yml

@@ -79,13 +79,13 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake libjemalloc-dev libhwloc-dev libnuma-dev libtbb-dev
+        sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
 
     - name: Install dependencies (ubuntu-20.04)
       if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake libjemalloc-dev libnuma-dev libtbb-dev
+        sudo apt-get install -y cmake libnuma-dev libtbb-dev
         .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package
 
     - name: Set ptrace value for IPC test (on Linux only)

.github/workflows/reusable_multi_numa.yml

@@ -45,7 +45,7 @@ jobs:
           -DUMF_BUILD_TESTS=ON
           -DUMF_DEVELOPER_MODE=ON
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
-          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
+          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=${{ matrix.os == 'rhel-9.1' && 'OFF' || 'ON' }}
           -DUMF_TESTS_FAIL_ON_SKIP=ON
           ${{ matrix.build_type == 'Debug' && matrix.os == 'ubuntu-22.04' && '-DUMF_USE_COVERAGE=ON' || '' }}
 

.github/workflows/reusable_proxy_lib.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Install apt packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y cmake libhwloc-dev libjemalloc-dev libtbb-dev lcov
+          sudo apt-get install -y cmake libhwloc-dev libtbb-dev lcov
 
       - name: Set ptrace value for IPC test
         run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"

.github/workflows/reusable_sanitizers.yml

@@ -29,7 +29,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
+        sudo apt-get install -y clang cmake libhwloc-dev libnuma-dev libtbb-dev
 
     - name: Install oneAPI basekit
       if: matrix.compiler.cxx == 'icpx'

.github/workflows/reusable_valgrind.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Install apt packages
       run: |
         sudo apt-get update
-        sudo apt-get install -y cmake hwloc libhwloc-dev libjemalloc-dev libnuma-dev libtbb-dev valgrind
+        sudo apt-get install -y cmake hwloc libhwloc-dev libnuma-dev libtbb-dev valgrind
 
     - name: Configure CMake
       run: >

CMakeLists.txt

@@ -111,6 +111,95 @@ else()
     message(FATAL_ERROR "Unknown OS type")
 endif()
 
+if(NOT UMF_BUILD_LIBUMF_POOL_JEMALLOC)
+    set(UMF_POOL_JEMALLOC_ENABLED FALSE)
+    set(JEMALLOC_FOUND FALSE)
+    set(JEMALLOC_LIBRARIES FALSE)
+elseif(WINDOWS)
+    pkg_check_modules(JEMALLOC jemalloc)
+    if(NOT JEMALLOC_FOUND)
+        find_package(JEMALLOC REQUIRED jemalloc)
+    endif()
+else()
+    if(NOT DEFINED UMF_JEMALLOC_REPO)
+        set(UMF_JEMALLOC_REPO "https://github.com/jemalloc/jemalloc.git")
+    endif()
+
+    if(NOT DEFINED UMF_JEMALLOC_TAG)
+        set(UMF_JEMALLOC_TAG 5.3.0)
+    endif()
+
+    include(FetchContent)
+    message(
+        STATUS
+            "Will fetch jemalloc from ${UMF_JEMALLOC_REPO} (tag: ${UMF_JEMALLOC_TAG})"
+    )
+
+    FetchContent_Declare(
+        jemalloc_targ
+        GIT_REPOSITORY ${UMF_JEMALLOC_REPO}
+        GIT_TAG ${UMF_JEMALLOC_TAG})
+    FetchContent_MakeAvailable(jemalloc_targ)
+
+    add_custom_command(
+        COMMAND ./autogen.sh
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_SOURCE_DIR}/configure)
+    add_custom_command(
+        # Custom jemalloc build. Non-default options used:
+        # --with-jemalloc-prefix=je_ - add je_ prefix to all public APIs
+        # --disable-cxx - Disable C++ integration. This will cause new and
+        # delete operators implementations to be omitted.
+        # --disable-initial-exec-tls - Disable the initial-exec TLS model for
+        # jemalloc's internal thread-local storage (on those platforms that
+        # support explicit settings). This can allow jemalloc to be dynamically
+        # loaded after program startup (e.g. using dlopen).
+        COMMAND
+            ./configure --prefix=${jemalloc_targ_BINARY_DIR}
+            --with-jemalloc-prefix=je_ --disable-cxx --disable-initial-exec-tls
+            CFLAGS=-fPIC
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_SOURCE_DIR}/Makefile
+        DEPENDS ${jemalloc_targ_SOURCE_DIR}/configure)
+    add_custom_command(
+        COMMAND make
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la
+        DEPENDS ${jemalloc_targ_SOURCE_DIR}/Makefile)
+    add_custom_command(
+        COMMAND make install
+        WORKING_DIRECTORY ${jemalloc_targ_SOURCE_DIR}
+        OUTPUT ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a
+        DEPENDS ${jemalloc_targ_SOURCE_DIR}/lib/libjemalloc.la)
+
+    add_custom_target(jemalloc_prod
+                      DEPENDS ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a)
+    add_library(jemalloc INTERFACE)
+    target_link_libraries(
+        jemalloc INTERFACE ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a)
+    add_dependencies(jemalloc jemalloc_prod)
+
+    set(JEMALLOC_LIBRARY_DIRS ${jemalloc_targ_BINARY_DIR}/lib)
+    set(JEMALLOC_INCLUDE_DIRS ${jemalloc_targ_BINARY_DIR}/include)
+    set(JEMALLOC_LIBRARIES ${jemalloc_targ_BINARY_DIR}/lib/libjemalloc.a)
+endif()
+
+if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES)
+    set(UMF_POOL_JEMALLOC_ENABLED TRUE)
+    # add PATH to DLL on Windows
+    set(DLL_PATH_LIST
+        "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}")
+    message(STATUS "    JEMALLOC_LIBRARIES = ${JEMALLOC_LIBRARIES}")
+    message(STATUS "    JEMALLOC_INCLUDE_DIRS = ${JEMALLOC_INCLUDE_DIRS}")
+    message(STATUS "    JEMALLOC_LIBRARY_DIRS = ${JEMALLOC_LIBRARY_DIRS}")
+else()
+    set(UMF_POOL_JEMALLOC_ENABLED FALSE)
+    message(
+        STATUS
+            "Disabling the Jemalloc Pool and tests and benchmarks that use it because jemalloc was not built/found."
+    )
+endif()
+
 if(UMF_DISABLE_HWLOC)
     message(STATUS "hwloc is disabled, hence OS provider, memtargets, "
                    "topology discovery, examples won't be available!")
@@ -402,19 +491,6 @@ else()
     set(UMF_POOL_SCALABLE_ENABLED FALSE)
 endif()
 
-if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
-    pkg_check_modules(JEMALLOC jemalloc)
-    if(NOT JEMALLOC_FOUND)
-        find_package(JEMALLOC REQUIRED jemalloc)
-    endif()
-    if(JEMALLOC_FOUND OR JEMALLOC_LIBRARIES)
-        set(UMF_POOL_JEMALLOC_ENABLED TRUE)
-        # add PATH to DLL on Windows
-        set(DLL_PATH_LIST
-            "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}")
-    endif()
-endif()
-
 if(WINDOWS)
     # TODO: enable the proxy library in the Debug build on Windows
     #
@@ -455,14 +531,14 @@ elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE)
         )
     endif()
 elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL JEMALLOC)
-    if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
+    if(UMF_POOL_JEMALLOC_ENABLED)
         set(UMF_PROXY_LIB_ENABLED ON)
         set(PROXY_LIB_USES_JEMALLOC_POOL ON)
-        set(PROXY_LIBS jemalloc_pool umf)
+        set(PROXY_LIBS umf)
     else()
         message(
             STATUS
-                "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but UMF_BUILD_LIBUMF_POOL_JEMALLOC is OFF"
+                "Disabling the proxy library, because UMF_PROXY_LIB_BASED_ON_POOL==JEMALLOC but the jemalloc pool is disabled"
         )
     endif()
 else()

README.md

@@ -281,11 +281,26 @@ pool manager built as a separate static library: libjemalloc_pool.a on Linux and
 jemalloc_pool.lib on Windows.
 The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option has to be turned `ON` to build this library.
 
+[jemalloc](https://github.com/jemalloc/jemalloc) is required to build the jemalloc pool.
+
+In case of Linux OS jemalloc is built from the (fetched) sources with the following
+non-default options enabled:
+- `--with-jemalloc-prefix=je_` - adds the `je_` prefix to all public APIs,
+- `--disable-cxx` - disables C++ integration, it will cause the `new` and the `delete`
+                    operators implementations to be omitted.
+- `--disable-initial-exec-tls` - disables the initial-exec TLS model for jemalloc's
+                    internal thread-local storage (on those platforms that support
+                    explicit settings), it can allow jemalloc to be dynamically
+                    loaded after program startup (e.g. using `dlopen()`).
+
+The default jemalloc package is required on Windows.
+
 ##### Requirements
 
 1) The `UMF_BUILD_LIBUMF_POOL_JEMALLOC` option turned `ON`
-2) Required packages:
-   - libjemalloc-dev (Linux) or jemalloc (Windows)
+2) jemalloc is required:
+- on Linux and MacOS: jemalloc is fetched and built from sources (a custom build),
+- on Windows: the default jemalloc package is required
 
 #### Scalable Pool (part of libumf)
 

benchmark/CMakeLists.txt

@@ -51,7 +51,7 @@ function(add_umf_benchmark)
 
     set(BENCH_NAME umf-${ARG_NAME})
 
-    set(BENCH_LIBS ${ARG_LIBS} umf)
+    set(BENCH_LIBS ${ARG_LIBS} umf umf_utils)
 
     add_umf_executable(
         NAME ${BENCH_NAME}
@@ -121,8 +121,7 @@ set(LIB_DIRS ${LIBHWLOC_LIBRARY_DIRS})
 if(UMF_BUILD_LIBUMF_POOL_DISJOINT)
     set(LIBS_OPTIONAL ${LIBS_OPTIONAL} disjoint_pool)
 endif()
-if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
-    set(LIBS_OPTIONAL ${LIBS_OPTIONAL} jemalloc_pool ${JEMALLOC_LIBRARIES})
+if(UMF_POOL_JEMALLOC_ENABLED)
     set(LIB_DIRS ${LIB_DIRS} ${JEMALLOC_LIBRARY_DIRS})
 endif()
 if(LINUX)

benchmark/multithread.cpp

@@ -139,11 +139,13 @@ int main() {
     // ctest looks for "PASSED" in the output
     std::cout << "PASSED" << std::endl;
 
+#if defined(UMF_POOL_DISJOINT_ENABLED)
     ret = umfDisjointPoolParamsDestroy(hDisjointParams);
     if (ret != UMF_RESULT_SUCCESS) {
         std::cerr << "disjoint pool params destroy failed" << std::endl;
         return -1;
     }
+#endif
 
     return 0;
 }

benchmark/ubench.c

@@ -445,8 +445,8 @@ static void do_ipc_get_put_benchmark(alloc_t *allocs, size_t num_allocs,
     }
 }
 
-int create_level_zero_params(ze_context_handle_t *context,
-                             ze_device_handle_t *device) {
+static int create_level_zero_params(ze_context_handle_t *context,
+                                    ze_device_handle_t *device) {
     uint32_t driver_idx = 0;
     ze_driver_handle_t driver = NULL;
 

examples/CMakeLists.txt

@@ -282,10 +282,13 @@ if(LINUX)
         add_umf_executable(
             NAME ${EXAMPLE_NAME}
             SRCS dram_and_fsdax/dram_and_fsdax.c
-            LIBS umf jemalloc_pool)
+            LIBS umf)
 
-        target_link_directories(${EXAMPLE_NAME} PRIVATE
-                                ${LIBHWLOC_LIBRARY_DIRS})
+        target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--no-as-needed,-ldl")
+
+        target_link_directories(
+            ${EXAMPLE_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS}
+            ${JEMALLOC_LIBRARY_DIRS})
 
         add_test(
             NAME ${EXAMPLE_NAME}

examples/cmake/FindJEMALLOC.cmake

@@ -2,9 +2,11 @@
 # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-message(STATUS "Checking for module 'jemalloc' using find_library()")
+message(
+    STATUS "Looking for the static 'libjemalloc.a' library using find_library()"
+)
 
-find_library(JEMALLOC_LIBRARY NAMES libjemalloc jemalloc)
+find_library(JEMALLOC_LIBRARY NAMES libjemalloc.a jemalloc.a)
 set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY})
 
 get_filename_component(JEMALLOC_LIB_DIR ${JEMALLOC_LIBRARIES} DIRECTORY)