full use of jemalloc with NUMA support and rework of thread spawning

build/jemalloc.m4

@@ -19,59 +19,71 @@ dnl jemalloc.m4: Trafficserver's jemalloc autoconf macros
 dnl
 
 AC_DEFUN([TS_CHECK_JEMALLOC], [
-enable_jemalloc=no
-AC_ARG_WITH([jemalloc], [AC_HELP_STRING([--with-jemalloc=DIR], [use a specific jemalloc library])],
-[
-  if test "$withval" != "no"; then
-    if test "x${enable_tcmalloc}" = "xyes"; then
-      AC_MSG_ERROR([Cannot compile with both jemalloc and tcmalloc])
-    fi
-    enable_jemalloc=yes
-    jemalloc_base_dir="$withval"
-    case "$withval" in
-      yes)
-        jemalloc_base_dir="/usr"
-        AC_MSG_CHECKING(checking for jemalloc includes standard directories)
-	;;
-      *":"*)
-        jemalloc_include="`echo $withval |sed -e 's/:.*$//'`"
-        jemalloc_ldflags="`echo $withval |sed -e 's/^.*://'`"
-        AC_MSG_CHECKING(checking for jemalloc includes in $jemalloc_include libs in $jemalloc_ldflags)
-        ;;
-      *)
-        jemalloc_include="$withval/include"
-        jemalloc_ldflags="$withval/lib"
-        AC_MSG_CHECKING(checking for jemalloc includes in $withval)
-        ;;
-    esac
-  fi
-])
+has_jemalloc=0
+AC_ARG_WITH([jemalloc], [AC_HELP_STRING([--with-jemalloc=DIR], [use a specific jemalloc library])])
+AS_IF([test -n "$with_jemalloc" -a "x$with_jemalloc" != "xno" -a "x$enable_tcmalloc" = "xyes"], 
+ [ AC_MSG_ERROR([Cannot compile with both jemalloc and tcmalloc]) ])
 
-jemalloch=0
-if test "$enable_jemalloc" != "no"; then
-  saved_ldflags=$LDFLAGS
-  saved_cppflags=$CPPFLAGS
-  jemalloc_have_headers=0
-  jemalloc_have_libs=0
-  if test "$jemalloc_base_dir" != "/usr"; then
-    TS_ADDTO(CPPFLAGS, [-I${jemalloc_include}])
-    TS_ADDTO(LDFLAGS, [-L${jemalloc_ldflags}])
-    TS_ADDTO_RPATH(${jemalloc_ldflags})
-  fi
-  # On Darwin, jemalloc symbols are prefixed with je_. Search for that first, then fall back
-  # to unadorned symbols.
-  AC_SEARCH_LIBS([je_malloc_stats_print], [jemalloc], [jemalloc_have_libs=1],
-    [AC_SEARCH_LIBS([malloc_stats_print], [jemalloc], [jemalloc_have_libs=1])]
-  )
-  if test "$jemalloc_have_libs" != "0"; then
-    AC_CHECK_HEADERS(jemalloc/jemalloc.h, [jemalloc_have_headers=1])
-  fi
-  if test "$jemalloc_have_headers" != "0"; then
-    jemalloch=1
-  else
-    CPPFLAGS=$saved_cppflags
-    LDFLAGS=$saved_ldflags
+AS_IF([test -n "$with_jemalloc" -a "x$with_jemalloc" != "xno" ],[
+  case "$withval" in
+    (yes)
+      PKG_CHECK_MODULES([JEMALLOC], [jemalloc >= 1.0], [have_jemalloc=yes], [:])
+      jemalloc_libdir="$($PKG_CONFIG jemalloc --variable=libdir)"
+      ;;
+    (*":"*)
+      jemalloc_incdir="$(echo $withval |sed -e 's/:.*$//')"
+      jemalloc_libdir="$(echo $withval |sed -e 's/^.*://')"
+      ;;
+    (*)
+      jemalloc_incdir="$withval/include"
+      jemalloc_libdir="$withval/lib"
+      ;;
+  esac
+
+  test -z "$jemalloc_libdir" -o -d "$jemalloc_libdir" || jemalloc_libdir=
+  test -z "$jemalloc_incdir" -o -d "$jemalloc_incdir" || jemalloc_incdir=
+
+  jemalloc_incdir="${jemalloc_incdir:+$(cd $jemalloc_incdir; pwd -P)}"
+  jemalloc_libdir="${jemalloc_libdir:+$(cd $jemalloc_libdir; pwd -P)}"
+
+  : ${JEMALLOC_CFLAGS:="${jemalloc_incdir:+ -I$jemalloc_incdir}"}
+  : ${JEMALLOC_LDFLAGS:="${jemalloc_libdir:+ -L$jemalloc_libdir}"}
+
+  save_cppflags="$CPPFLAGS"
+  save_ldflags="$LDFLAGS"
+  LDFLAGS="$LDFLAGS $JEMALLOC_LDFLAGS"
+  CPPFLAGS="$CPPFLAGS $JEMALLOC_CFLAGS"
+
+dnl
+dnl define HAVE_LIBJEMALLOC is to be set and -ljemalloc added into LIBS?
+dnl 
+  AC_CHECK_LIB([jemalloc],[je_malloc_stats_print],[], 
+    [AC_CHECK_LIB([jemalloc],[malloc_stats_print],[],
+    [have_jemalloc=no])])
+
+dnl
+dnl define HAVE_JEMALLOC_JEMALLOC_H is to be set?
+dnl define HAVE_JEMALLOC_H is to be set?
+dnl 
+  AC_CHECK_HEADERS(jemalloc/jemalloc.h, [], 
+    [AC_CHECK_HEADERS(jemalloc.h, [], [have_jemalloc=no])])
+
+  LDFLAGS="$save_ldflags"
+  CPPFLAGS="$save_cppflags"
+  LIBS="$(echo "$LIBS" | sed -e 's/ *-ljemalloc//' -e 's/$/ -ljemalloc/')"
+
+  AS_IF([test "x$have_jemalloc" == "xno" ], 
+     [AC_MSG_ERROR([Failed to compile with jemalloc.h and -ljemalloc]) ])
+
+  has_jemalloc=1
+#
+# flags must be global setting for all libs
+# 
+  TS_ADDTO(CPPFLAGS,$JEMALLOC_CFLAGS)
+  if test -n "$jemalloc_libdir"; then
+      TS_ADDTO_RPATH($jemalloc_libdir)
+      TS_ADDTO(LDFLAGS,$JEMALLOC_LDFLAGS)
   fi
-fi
-AC_SUBST(jemalloch)
+])
+AC_SUBST([has_jemalloc])
 ])

iocore/eventsystem/I_ProxyAllocator.h

@@ -31,7 +31,13 @@
 #ifndef _I_ProxyAllocator_h_
 #define _I_ProxyAllocator_h_
 
-#include "ts/ink_platform.h"
+#include "ts/ink_assert.h"
+
+#include "ts/Allocator.h"
+
+#ifdef ProxyAllocator
+#error "cannot compile together with new JEMalloc includes"
+#endif
 
 class EThread;
 

iocore/eventsystem/I_Thread.h

@@ -63,11 +63,15 @@
 #error "include I_EventSystem.h or P_EventSystem.h"
 #endif
 
+#include "ts/Allocator.h"
+#if !HAVE_LIBJEMALLOC
+#include "I_ProxyAllocator.h"
+#endif
+
 #include <functional>
 
 #include "ts/ink_platform.h"
 #include "ts/ink_thread.h"
-#include "I_ProxyAllocator.h"
 
 class ProxyMutex;
 

iocore/eventsystem/ProxyAllocator.cc

@@ -20,11 +20,17 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 */
-#include "I_EventSystem.h"
-
 int thread_freelist_high_watermark = 512;
 int thread_freelist_low_watermark  = 32;
 
+// must include for defines
+#include "ts/ink_config.h"
+
+#undef HAVE_LIBJEMALLOC
+
+// safe to use older system
+#include "I_ProxyAllocator.h"
+
 void *
 thread_alloc(Allocator &a, ProxyAllocator &l)
 {

lib/ts/Allocator.h

@@ -49,6 +49,39 @@
 
 #define RND16(_x) (((_x) + 15) & ~15)
 
+struct _InkFreeList {
+  volatile head_p head;
+  const char *name;
+  uint32_t type_size, chunk_size, used, allocated, alignment;
+  uint32_t allocated_base, used_base;
+  int advice;
+};
+
+typedef struct ink_freelist_ops InkFreeListOps;
+typedef struct _InkFreeList InkFreeList;
+
+extern "C" {
+
+const InkFreeListOps *ink_freelist_malloc_ops();
+const InkFreeListOps *ink_freelist_freelist_ops();
+void ink_freelist_init_ops(const InkFreeListOps *);
+
+/*
+ * alignment must be a power of 2
+ */
+InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment);
+
+inkcoreapi void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment);
+inkcoreapi void ink_freelist_madvise_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size,
+                                          uint32_t alignment, int advice);
+inkcoreapi void *ink_freelist_new(InkFreeList *f);
+inkcoreapi void ink_freelist_free(InkFreeList *f, void *item);
+inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item);
+void ink_freelists_dump(FILE *f);
+void ink_freelists_dump_baselinerel(FILE *f);
+void ink_freelists_snap_baseline();
+}
+
 /** Allocator for fixed size memory blocks. */
 class Allocator
 {
@@ -252,4 +285,34 @@ template <class C> class TrackerClassAllocator : public ClassAllocator<C>
   ink_mutex trackerLock;
 };
 
+#if HAVE_LIBJEMALLOC
+
+// these calls can be removed
+#define ink_freelists_dump(...)
+#define ink_freelists_dump_baselinerel(...)
+#define ink_freelists_snap_baseline(...)
+
+#define ink_freelist_init_ops(...)
+
+#include "ts/StdAllocWrapper.h"
+
+// cover up use of original Allocators above
+#define Allocator AlignedAllocator
+#define ClassAllocator ObjAllocator
+#define ProxyAllocator ThreadAllocatorStub
+
+class ThreadAllocatorStub
+{
+};
+
+// define use of for thread Allocators
+#define THREAD_ALLOC(allocObj, t) (::allocObj.alloc())
+#define THREAD_ALLOC_INIT(allocObj, t) (::allocObj.alloc())
+#define THREAD_FREE(ptr, allocObj, t) (::allocObj.free(ptr))
+
+extern int thread_freelist_high_watermark;
+extern int thread_freelist_low_watermark;
+
+#endif
+
 #endif // _Allocator_h_

lib/ts/Arena.cc

@@ -23,8 +23,13 @@
 
 #include "ts/ink_platform.h"
 #include "ts/ink_memory.h"
+
+// need standard Allocator for this
+#undef HAVE_LIBJEMALLOC
+
 #include "ts/Allocator.h"
 #include "ts/Arena.h"
+
 #include <cassert>
 #include <cstring>
 

lib/ts/Arena.h

@@ -23,10 +23,11 @@
 
 #ifndef __ARENA_H__
 #define __ARENA_H__
+#include "ts/ink_assert.h"
+#include "ts/ink_memory.h"
 
 #include <sys/types.h>
 #include <memory.h>
-#include "ts/ink_assert.h"
 
 struct ArenaBlock {
   ArenaBlock *next;
@@ -166,4 +167,23 @@ Arena::str_store(const char *str, size_t len)
   return mem;
 }
 
+struct Arena_NoCache : public Arena {
+  std::allocator<uint64_t> m_alloc;
+
+  inkcoreapi void *
+  alloc(size_t size, size_t alignment = sizeof(double))
+  {
+    return m_alloc.allocate((size + 1) / sizeof(uint64_t));
+  }
+  void
+  free(void *mem, size_t size)
+  {
+    m_alloc.deallocate(static_cast<uint64_t *>(mem), size);
+  }
+};
+
+#if HAVE_LIBJEMALLOC
+#define Arena Arena_NoCache
+#endif
+
 #endif /* __ARENA_H__ */

lib/ts/Makefile.am

@@ -41,11 +41,14 @@ libtsutil_la_LIBADD = \
   @OPENSSL_LIBS@ \
   @LIBTCL@ \
   @LIBRESOLV@ \
-  @LIBCAP@ \
-  -lc
+  @LIBCAP@
 
 libtsutil_la_SOURCES = \
+  jemallctl.cc \
+  jemallctl.h \
   Allocator.h \
+  StdAllocWrapper.h \
+  StdAllocWrapper.cc \
   Arena.cc \
   Arena.h \
   BaseLogFile.cc \
@@ -255,6 +258,7 @@ test_tslib_SOURCES = \
 	unit-tests/string_view.cpp
 
 CompileParseRules_SOURCES = CompileParseRules.cc
+CompileParseRules_LDADD = ;true            # link command doesn't need $(LIBS)
 
 clean-local:
 	rm -f ParseRulesCType ParseRulesCTypeToLower ParseRulesCTypeToUpper

lib/ts/StdAllocWrapper.cc

@@ -0,0 +1,61 @@
+/** @file
+
+  A brief file description
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include "StdAllocWrapper.h"
+#include "ts/jemallctl.h"
+
+#include "ts/ink_queue.h"
+#include "ts/ink_defs.h"
+#include "ts/ink_resource.h"
+#include "ts/ink_align.h"
+#include "ts/ink_memory.h"
+
+void
+AlignedAllocator::re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, int advice)
+{
+  _name = name;
+  _sz   = aligned_spacing(element_size, std::max(sizeof(uint64_t), alignment + 0UL)); // increase to aligned size
+
+  if (advice == MADV_DONTDUMP) {
+    _arena = jemallctl::proc_arena_nodump;
+  } else if (advice == MADV_NORMAL) {
+    _arena = jemallctl::proc_arena;
+  } else {
+    ink_abort("allocator re_init: unknown madvise() flags: %x", advice);
+  }
+
+  void *preCached[chunk_size];
+
+  for (int n = chunk_size; n--;) {
+    preCached[n] = mallocx(_sz, (MALLOCX_ALIGN(_sz) | MALLOCX_ARENA(_arena)));
+  }
+  for (int n = chunk_size; n--;) {
+    deallocate(preCached[n]);
+  }
+}
+
+AlignedAllocator::AlignedAllocator(const char *name, unsigned int element_size)
+  : _name(name), _sz(aligned_spacing(element_size, sizeof(uint64_t)))
+{
+  // don't pre-allocate before main() is called
+}