diff --git a/build/jemalloc.m4 b/build/jemalloc.m4 index 3a4ed480212..75aaf34464c 100644 --- a/build/jemalloc.m4 +++ b/build/jemalloc.m4 @@ -19,59 +19,71 @@ dnl jemalloc.m4: Trafficserver's jemalloc autoconf macros dnl AC_DEFUN([TS_CHECK_JEMALLOC], [ -enable_jemalloc=no -AC_ARG_WITH([jemalloc], [AC_HELP_STRING([--with-jemalloc=DIR], [use a specific jemalloc library])], -[ - if test "$withval" != "no"; then - if test "x${enable_tcmalloc}" = "xyes"; then - AC_MSG_ERROR([Cannot compile with both jemalloc and tcmalloc]) - fi - enable_jemalloc=yes - jemalloc_base_dir="$withval" - case "$withval" in - yes) - jemalloc_base_dir="/usr" - AC_MSG_CHECKING(checking for jemalloc includes standard directories) - ;; - *":"*) - jemalloc_include="`echo $withval |sed -e 's/:.*$//'`" - jemalloc_ldflags="`echo $withval |sed -e 's/^.*://'`" - AC_MSG_CHECKING(checking for jemalloc includes in $jemalloc_include libs in $jemalloc_ldflags) - ;; - *) - jemalloc_include="$withval/include" - jemalloc_ldflags="$withval/lib" - AC_MSG_CHECKING(checking for jemalloc includes in $withval) - ;; - esac - fi -]) +has_jemalloc=0 +AC_ARG_WITH([jemalloc], [AC_HELP_STRING([--with-jemalloc=DIR], [use a specific jemalloc library])]) +AS_IF([test -n "$with_jemalloc" -a "x$with_jemalloc" != "xno" -a "x$enable_tcmalloc" = "xyes"], + [ AC_MSG_ERROR([Cannot compile with both jemalloc and tcmalloc]) ]) -jemalloch=0 -if test "$enable_jemalloc" != "no"; then - saved_ldflags=$LDFLAGS - saved_cppflags=$CPPFLAGS - jemalloc_have_headers=0 - jemalloc_have_libs=0 - if test "$jemalloc_base_dir" != "/usr"; then - TS_ADDTO(CPPFLAGS, [-I${jemalloc_include}]) - TS_ADDTO(LDFLAGS, [-L${jemalloc_ldflags}]) - TS_ADDTO_RPATH(${jemalloc_ldflags}) - fi - # On Darwin, jemalloc symbols are prefixed with je_. Search for that first, then fall back - # to unadorned symbols. - AC_SEARCH_LIBS([je_malloc_stats_print], [jemalloc], [jemalloc_have_libs=1], - [AC_SEARCH_LIBS([malloc_stats_print], [jemalloc], [jemalloc_have_libs=1])] - ) - if test "$jemalloc_have_libs" != "0"; then - AC_CHECK_HEADERS(jemalloc/jemalloc.h, [jemalloc_have_headers=1]) - fi - if test "$jemalloc_have_headers" != "0"; then - jemalloch=1 - else - CPPFLAGS=$saved_cppflags - LDFLAGS=$saved_ldflags +AS_IF([test -n "$with_jemalloc" -a "x$with_jemalloc" != "xno" ],[ + case "$withval" in + (yes) + PKG_CHECK_MODULES([JEMALLOC], [jemalloc >= 1.0], [have_jemalloc=yes], [:]) + jemalloc_libdir="$($PKG_CONFIG jemalloc --variable=libdir)" + ;; + (*":"*) + jemalloc_incdir="$(echo $withval |sed -e 's/:.*$//')" + jemalloc_libdir="$(echo $withval |sed -e 's/^.*://')" + ;; + (*) + jemalloc_incdir="$withval/include" + jemalloc_libdir="$withval/lib" + ;; + esac + + test -z "$jemalloc_libdir" -o -d "$jemalloc_libdir" || jemalloc_libdir= + test -z "$jemalloc_incdir" -o -d "$jemalloc_incdir" || jemalloc_incdir= + + jemalloc_incdir="${jemalloc_incdir:+$(cd $jemalloc_incdir; pwd -P)}" + jemalloc_libdir="${jemalloc_libdir:+$(cd $jemalloc_libdir; pwd -P)}" + + : ${JEMALLOC_CFLAGS:="${jemalloc_incdir:+ -I$jemalloc_incdir}"} + : ${JEMALLOC_LDFLAGS:="${jemalloc_libdir:+ -L$jemalloc_libdir}"} + + save_cppflags="$CPPFLAGS" + save_ldflags="$LDFLAGS" + LDFLAGS="$LDFLAGS $JEMALLOC_LDFLAGS" + CPPFLAGS="$CPPFLAGS $JEMALLOC_CFLAGS" + +dnl +dnl define HAVE_LIBJEMALLOC is to be set and -ljemalloc added into LIBS? +dnl + AC_CHECK_LIB([jemalloc],[je_malloc_stats_print],[], + [AC_CHECK_LIB([jemalloc],[malloc_stats_print],[], + [have_jemalloc=no])]) + +dnl +dnl define HAVE_JEMALLOC_JEMALLOC_H is to be set? +dnl define HAVE_JEMALLOC_H is to be set? +dnl + AC_CHECK_HEADERS(jemalloc/jemalloc.h, [], + [AC_CHECK_HEADERS(jemalloc.h, [], [have_jemalloc=no])]) + + LDFLAGS="$save_ldflags" + CPPFLAGS="$save_cppflags" + LIBS="$(echo "$LIBS" | sed -e 's/ *-ljemalloc//' -e 's/$/ -ljemalloc/')" + + AS_IF([test "x$have_jemalloc" == "xno" ], + [AC_MSG_ERROR([Failed to compile with jemalloc.h and -ljemalloc]) ]) + + has_jemalloc=1 +# +# flags must be global setting for all libs +# + TS_ADDTO(CPPFLAGS,$JEMALLOC_CFLAGS) + if test -n "$jemalloc_libdir"; then + TS_ADDTO_RPATH($jemalloc_libdir) + TS_ADDTO(LDFLAGS,$JEMALLOC_LDFLAGS) fi -fi -AC_SUBST(jemalloch) +]) +AC_SUBST([has_jemalloc]) ]) diff --git a/iocore/eventsystem/I_ProxyAllocator.h b/iocore/eventsystem/I_ProxyAllocator.h index 3650f2a2e7b..791ecc8e78f 100644 --- a/iocore/eventsystem/I_ProxyAllocator.h +++ b/iocore/eventsystem/I_ProxyAllocator.h @@ -31,7 +31,13 @@ #ifndef _I_ProxyAllocator_h_ #define _I_ProxyAllocator_h_ -#include "ts/ink_platform.h" +#include "ts/ink_assert.h" + +#include "ts/Allocator.h" + +#ifdef ProxyAllocator +#error "cannot compile together with new JEMalloc includes" +#endif class EThread; diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h index f053dec1633..954e9372d82 100644 --- a/iocore/eventsystem/I_Thread.h +++ b/iocore/eventsystem/I_Thread.h @@ -63,11 +63,15 @@ #error "include I_EventSystem.h or P_EventSystem.h" #endif +#include "ts/Allocator.h" +#if !HAVE_LIBJEMALLOC +#include "I_ProxyAllocator.h" +#endif + #include #include "ts/ink_platform.h" #include "ts/ink_thread.h" -#include "I_ProxyAllocator.h" class ProxyMutex; diff --git a/iocore/eventsystem/ProxyAllocator.cc b/iocore/eventsystem/ProxyAllocator.cc index e91aaaba6f1..e0df732b471 100644 --- a/iocore/eventsystem/ProxyAllocator.cc +++ b/iocore/eventsystem/ProxyAllocator.cc @@ -20,11 +20,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include "I_EventSystem.h" - int thread_freelist_high_watermark = 512; int thread_freelist_low_watermark = 32; +// must include for defines +#include "ts/ink_config.h" + +#undef HAVE_LIBJEMALLOC + +// safe to use older system +#include "I_ProxyAllocator.h" + void * thread_alloc(Allocator &a, ProxyAllocator &l) { diff --git a/lib/ts/Allocator.h b/lib/ts/Allocator.h index 0046f9387fa..c7ca8a0e2c3 100644 --- a/lib/ts/Allocator.h +++ b/lib/ts/Allocator.h @@ -49,6 +49,39 @@ #define RND16(_x) (((_x) + 15) & ~15) +struct _InkFreeList { + volatile head_p head; + const char *name; + uint32_t type_size, chunk_size, used, allocated, alignment; + uint32_t allocated_base, used_base; + int advice; +}; + +typedef struct ink_freelist_ops InkFreeListOps; +typedef struct _InkFreeList InkFreeList; + +extern "C" { + +const InkFreeListOps *ink_freelist_malloc_ops(); +const InkFreeListOps *ink_freelist_freelist_ops(); +void ink_freelist_init_ops(const InkFreeListOps *); + +/* + * alignment must be a power of 2 + */ +InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment); + +inkcoreapi void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment); +inkcoreapi void ink_freelist_madvise_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, + uint32_t alignment, int advice); +inkcoreapi void *ink_freelist_new(InkFreeList *f); +inkcoreapi void ink_freelist_free(InkFreeList *f, void *item); +inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item); +void ink_freelists_dump(FILE *f); +void ink_freelists_dump_baselinerel(FILE *f); +void ink_freelists_snap_baseline(); +} + /** Allocator for fixed size memory blocks. */ class Allocator { @@ -252,4 +285,34 @@ template class TrackerClassAllocator : public ClassAllocator ink_mutex trackerLock; }; +#if HAVE_LIBJEMALLOC + +// these calls can be removed +#define ink_freelists_dump(...) +#define ink_freelists_dump_baselinerel(...) +#define ink_freelists_snap_baseline(...) + +#define ink_freelist_init_ops(...) + +#include "ts/StdAllocWrapper.h" + +// cover up use of original Allocators above +#define Allocator AlignedAllocator +#define ClassAllocator ObjAllocator +#define ProxyAllocator ThreadAllocatorStub + +class ThreadAllocatorStub +{ +}; + +// define use of for thread Allocators +#define THREAD_ALLOC(allocObj, t) (::allocObj.alloc()) +#define THREAD_ALLOC_INIT(allocObj, t) (::allocObj.alloc()) +#define THREAD_FREE(ptr, allocObj, t) (::allocObj.free(ptr)) + +extern int thread_freelist_high_watermark; +extern int thread_freelist_low_watermark; + +#endif + #endif // _Allocator_h_ diff --git a/lib/ts/Arena.cc b/lib/ts/Arena.cc index daa52b4f61f..40f72c00129 100644 --- a/lib/ts/Arena.cc +++ b/lib/ts/Arena.cc @@ -23,8 +23,13 @@ #include "ts/ink_platform.h" #include "ts/ink_memory.h" + +// need standard Allocator for this +#undef HAVE_LIBJEMALLOC + #include "ts/Allocator.h" #include "ts/Arena.h" + #include #include diff --git a/lib/ts/Arena.h b/lib/ts/Arena.h index cd5ad688459..ad4db83393b 100644 --- a/lib/ts/Arena.h +++ b/lib/ts/Arena.h @@ -23,10 +23,11 @@ #ifndef __ARENA_H__ #define __ARENA_H__ +#include "ts/ink_assert.h" +#include "ts/ink_memory.h" #include #include -#include "ts/ink_assert.h" struct ArenaBlock { ArenaBlock *next; @@ -166,4 +167,23 @@ Arena::str_store(const char *str, size_t len) return mem; } +struct Arena_NoCache : public Arena { + std::allocator m_alloc; + + inkcoreapi void * + alloc(size_t size, size_t alignment = sizeof(double)) + { + return m_alloc.allocate((size + 1) / sizeof(uint64_t)); + } + void + free(void *mem, size_t size) + { + m_alloc.deallocate(static_cast(mem), size); + } +}; + +#if HAVE_LIBJEMALLOC +#define Arena Arena_NoCache +#endif + #endif /* __ARENA_H__ */ diff --git a/lib/ts/Makefile.am b/lib/ts/Makefile.am index 737dbfedafa..60a154651d6 100644 --- a/lib/ts/Makefile.am +++ b/lib/ts/Makefile.am @@ -41,11 +41,14 @@ libtsutil_la_LIBADD = \ @OPENSSL_LIBS@ \ @LIBTCL@ \ @LIBRESOLV@ \ - @LIBCAP@ \ - -lc + @LIBCAP@ libtsutil_la_SOURCES = \ + jemallctl.cc \ + jemallctl.h \ Allocator.h \ + StdAllocWrapper.h \ + StdAllocWrapper.cc \ Arena.cc \ Arena.h \ BaseLogFile.cc \ @@ -255,6 +258,7 @@ test_tslib_SOURCES = \ unit-tests/string_view.cpp CompileParseRules_SOURCES = CompileParseRules.cc +CompileParseRules_LDADD = ;true # link command doesn't need $(LIBS) clean-local: rm -f ParseRulesCType ParseRulesCTypeToLower ParseRulesCTypeToUpper diff --git a/lib/ts/StdAllocWrapper.cc b/lib/ts/StdAllocWrapper.cc new file mode 100644 index 00000000000..c6b93e2f7cb --- /dev/null +++ b/lib/ts/StdAllocWrapper.cc @@ -0,0 +1,61 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "StdAllocWrapper.h" +#include "ts/jemallctl.h" + +#include "ts/ink_queue.h" +#include "ts/ink_defs.h" +#include "ts/ink_resource.h" +#include "ts/ink_align.h" +#include "ts/ink_memory.h" + +void +AlignedAllocator::re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, int advice) +{ + _name = name; + _sz = aligned_spacing(element_size, std::max(sizeof(uint64_t), alignment + 0UL)); // increase to aligned size + + if (advice == MADV_DONTDUMP) { + _arena = jemallctl::proc_arena_nodump; + } else if (advice == MADV_NORMAL) { + _arena = jemallctl::proc_arena; + } else { + ink_abort("allocator re_init: unknown madvise() flags: %x", advice); + } + + void *preCached[chunk_size]; + + for (int n = chunk_size; n--;) { + preCached[n] = mallocx(_sz, (MALLOCX_ALIGN(_sz) | MALLOCX_ARENA(_arena))); + } + for (int n = chunk_size; n--;) { + deallocate(preCached[n]); + } +} + +AlignedAllocator::AlignedAllocator(const char *name, unsigned int element_size) + : _name(name), _sz(aligned_spacing(element_size, sizeof(uint64_t))) +{ + // don't pre-allocate before main() is called +} diff --git a/lib/ts/StdAllocWrapper.h b/lib/ts/StdAllocWrapper.h new file mode 100644 index 00000000000..bdb40a69d47 --- /dev/null +++ b/lib/ts/StdAllocWrapper.h @@ -0,0 +1,159 @@ +/** @file + + Fast-Allocators + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Provides three classes + - AlignedAllocator for allocating memory blocks of fixed size / alignment + - ObjAllocator for allocating objects + + These class provides a efficient way for handling dynamic allocation. + The fast allocator maintains its own freepool of objects from + which it doles out object. Allocated objects when freed go back + to the free pool. + + @note Fast allocators could accumulate a lot of objects in the + free pool as a result of bursty demand. Memory used by the objects + in the free pool never gets freed even if the freelist grows very + large. + + */ + +#pragma once + +#include "ts/jemallctl.h" + +#include "ts/ink_queue.h" +#include "ts/ink_defs.h" +#include "ts/ink_resource.h" +#include "ts/ink_align.h" +#include "ts/ink_memory.h" + +#include // for backtrace! + +#include +#include +#include + +class AlignedAllocator +{ + const char *_name = nullptr; + size_t _sz = 0; // bytes and alignment (both) + size_t _arena = 0; // jemalloc arena + +public: + AlignedAllocator() {} + AlignedAllocator(const char *name, unsigned int element_size); + + void * + alloc_void() + { + return allocate(); + } + void + free_void(void *ptr) + { + deallocate(ptr); + } + void * + alloc() + { + return alloc_void(); + } + void + free(void *ptr) + { + free_void(ptr); + } + + void re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, int advice); + +protected: + void * + allocate() + { + return mallocx(_sz, (MALLOCX_ALIGN(_sz) | MALLOCX_ZERO | MALLOCX_ARENA(_arena))); + } + void + deallocate(void *p) + { + dallocx(p, MALLOCX_ARENA(_arena)); + } +}; + +template class ObjAllocator : public std::allocator +{ +public: + using typename std::allocator::value_type; + + ObjAllocator(const char *name, unsigned chunk_size = 128) : _name(name) + { + value_type *preCached[chunk_size]; + + for (int n = chunk_size; n--;) { + // create correct size and alignment + preCached[n] = static_cast(mallocx(sizeof(value_type), MALLOCX_ALIGN(alignof(value_type)))); + } + for (int n = chunk_size; n--;) { + deallocate(preCached[n]); + } + } + + void * + alloc_void() + { + return allocate(); + } + void + free_void(void *ptr) + { + static_cast(ptr)->~value_type(); + deallocate(ptr); + } + value_type * + alloc() + { + return allocate(); + } + void + free(value_type *ptr) + { + ptr->~value_type(); + deallocate(ptr); + } + +protected: + value_type * + allocate() + { + auto p = static_cast(mallocx(sizeof(value_type), MALLOCX_ALIGN(alignof(value_type)) | MALLOCX_ZERO)); + this->construct(p); // default constructor + pre-zeroed + return p; + } + + void + deallocate(value_type *p) + { + sdallocx(p, sizeof(value_type), 0); + } + +private: + const char *_name; +}; diff --git a/lib/ts/ink_align.h b/lib/ts/ink_align.h index 591b311e460..5e79dd89956 100644 --- a/lib/ts/ink_align.h +++ b/lib/ts/ink_align.h @@ -53,6 +53,12 @@ union Alias64 { /** Default alignment */ #define INK_ALIGN_DEFAULT(size) INK_ALIGN(size, INK_MIN_ALIGN) +static inline size_t +aligned_spacing(size_t len, size_t block = INK_MIN_ALIGN) +{ + return INK_ALIGN(len, block); +} + // // Move a pointer forward until it meets the alignment width. // diff --git a/lib/ts/ink_config.h.in b/lib/ts/ink_config.h.in index 79b2c00c395..3294813d684 100644 --- a/lib/ts/ink_config.h.in +++ b/lib/ts/ink_config.h.in @@ -51,7 +51,7 @@ #define BUILD_NUMBER "@build_number@" /* Libraries */ -#define TS_HAS_JEMALLOC @jemalloch@ +#define TS_HAS_JEMALLOC @has_jemalloc@ #define TS_HAS_TCMALLOC @has_tcmalloc@ /* Features */ diff --git a/lib/ts/ink_memory.h b/lib/ts/ink_memory.h index 8968fa8fd62..ae855b98667 100644 --- a/lib/ts/ink_memory.h +++ b/lib/ts/ink_memory.h @@ -23,6 +23,11 @@ #ifndef _ink_memory_h_ #define _ink_memory_h_ +#include "ts/ink_config.h" +#include "ts/ink_defs.h" + +#include + #include #include #include @@ -48,13 +53,23 @@ #include #endif -#if TS_HAS_JEMALLOC +#if HAVE_JEMALLOC_JEMALLOC_H #include -#else +#elif HAVE_JEMALLOC_H +#include + +#else // no jemalloc includes used + +#define mallocx(...) nullptr +#define sallocx(...) \ + size_t {} +#define sdallocx(...) +#define dallocx(...) + #if HAVE_MALLOC_H #include -#endif // ! HAVE_MALLOC_H -#endif // ! TS_HAS_JEMALLOC +#endif +#endif // ! HAVE_JEMALLOC_H && ! HAVE_JEMALLOC_JEMALLOC_H #ifndef MADV_NORMAL #define MADV_NORMAL 0 diff --git a/lib/ts/ink_queue.cc b/lib/ts/ink_queue.cc index 844a67cdd97..f6eee9587f0 100644 --- a/lib/ts/ink_queue.cc +++ b/lib/ts/ink_queue.cc @@ -52,6 +52,11 @@ #include "ts/hugepages.h" #include "ts/Diags.h" + +// in order to compile correctly +#undef HAVE_LIBJEMALLOC +#include "ts/Allocator.h" + #define DEBUG_TAG "freelist" /* diff --git a/lib/ts/ink_queue.h b/lib/ts/ink_queue.h index 0fcc90e54a0..288e7326389 100644 --- a/lib/ts/ink_queue.h +++ b/lib/ts/ink_queue.h @@ -146,36 +146,6 @@ union head_p { #error "unsupported processor" #endif -struct _InkFreeList { - volatile head_p head; - const char *name; - uint32_t type_size, chunk_size, used, allocated, alignment; - uint32_t allocated_base, used_base; - int advice; -}; - -typedef struct ink_freelist_ops InkFreeListOps; -typedef struct _InkFreeList InkFreeList; - -const InkFreeListOps *ink_freelist_malloc_ops(); -const InkFreeListOps *ink_freelist_freelist_ops(); -void ink_freelist_init_ops(const InkFreeListOps *); - -/* - * alignment must be a power of 2 - */ -InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment); - -inkcoreapi void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment); -inkcoreapi void ink_freelist_madvise_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, - uint32_t alignment, int advice); -inkcoreapi void *ink_freelist_new(InkFreeList *f); -inkcoreapi void ink_freelist_free(InkFreeList *f, void *item); -inkcoreapi void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item); -void ink_freelists_dump(FILE *f); -void ink_freelists_dump_baselinerel(FILE *f); -void ink_freelists_snap_baseline(); - struct InkAtomicList { InkAtomicList() {} volatile head_p head{}; diff --git a/lib/ts/jemallctl.cc b/lib/ts/jemallctl.cc new file mode 100644 index 00000000000..622cc887eb1 --- /dev/null +++ b/lib/ts/jemallctl.cc @@ -0,0 +1,261 @@ +/** @file + + Memory allocation routines for libts + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "ts/jemallctl.h" + +#include "ts/ink_platform.h" + +// includes jemalloc.h +#include "ts/ink_memory.h" +#include "ts/ink_defs.h" +#include "ts/ink_stack_trace.h" +#include "ts/Diags.h" +#include "ts/ink_atomic.h" +#include "ts/ink_align.h" + +#include +#if defined(linux) && !defined(_XOPEN_SOURCE) +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include + +#include + +namespace jemallctl +{ +// internal read/write functions + +int mallctl_void(const objpath_t &oid); + +template auto mallctl_get(const objpath_t &oid) -> T_VALUE; + +template auto mallctl_set(const objpath_t &oid, const T_VALUE &v) -> int; + +// define object functors (to allow instances below) + +objpath_t objpath(const std::string &path); + +ObjBase::ObjBase(const char *name) : _oid(objpath(name)) +{ +} + +template +auto +GetObjFxn::operator()(void) const -> T_VALUE +{ + return std::move(::jemallctl::mallctl_get(ObjBase::_oid)); +} + +template +auto +SetObjFxn::operator()(const T_VALUE &v) const -> int +{ + return ::jemallctl::mallctl_set(ObjBase::_oid, v); +} + +int +GetObjFxn::operator()(void) const +{ + return ::jemallctl::mallctl_void(ObjBase::_oid); +} + +#if !HAVE_LIBJEMALLOC +objpath_t +objpath(const std::string &path) +{ + return objpath_t(); +} +auto +mallctl_void(const objpath_t &oid) -> int +{ + return -1; +} +template +auto +mallctl_set(const objpath_t &oid, const T_VALUE &v) -> int +{ + return -1; +} +template +auto +mallctl_get(const objpath_t &oid) -> T_VALUE +{ + return std::move(T_VALUE{}); +} + +#else + +objpath_t +objpath(const std::string &path) +{ + objpath_t oid; + + oid.resize(10); // longer than any oid target// *explicitly resize* + size_t len = oid.size(); + mallctlnametomib(path.c_str(), oid.data(), &len); + oid.resize(len); + return std::move(oid); +} + +/// implementations for ObjBase + +auto +mallctl_void(const objpath_t &oid) -> int +{ + return mallctlbymib(oid.data(), oid.size(), nullptr, nullptr, nullptr, 0); +} + +template +auto +mallctl_set(const objpath_t &oid, const T_VALUE &v) -> int +{ + return mallctlbymib(oid.data(), oid.size(), nullptr, nullptr, const_cast(&v), sizeof(v)); +} + +template +auto +mallctl_get(const objpath_t &oid) -> T_VALUE +{ + T_VALUE v{}; // init to zero if a pod type + size_t len = sizeof(v); + mallctlbymib(oid.data(), oid.size(), &v, &len, nullptr, 0); + return std::move(v); +} + +template <> +auto +mallctl_get(const objpath_t &oid) -> std::string +{ + char buff[256]; // adequate for paths and most things + size_t len = sizeof(buff); + mallctlbymib(oid.data(), oid.size(), &buff, &len, nullptr, 0); + std::string v(&buff[0], len); // copy out + return std::move(v); +} + +template <> +auto +mallctl_set(const objpath_t &oid, const std::string &v) -> int +{ + return mallctlbymib(oid.data(), oid.size(), nullptr, nullptr, const_cast(v.c_str()), v.size()); +} + +template <> +auto +mallctl_get(const objpath_t &baseOid) -> chunk_hooks_t +{ + objpath_t oid = baseOid; + oid[1] = thread_arena(); + + chunk_hooks_t v; + size_t len = sizeof(v); + mallctlbymib(oid.data(), oid.size(), &v, &len, nullptr, 0); + return std::move(v); +} + +template <> +auto +mallctl_set(const objpath_t &baseOid, const chunk_hooks_t &hooks) -> int +{ + objpath_t oid = baseOid; + oid[1] = ::jemallctl::thread_arena(); + auto ohooks = mallctl_get(oid); + auto nhooks = chunk_hooks_t{(hooks.alloc ?: ohooks.alloc), (hooks.dalloc ?: ohooks.dalloc), (hooks.commit ?: ohooks.commit), + (hooks.decommit ?: ohooks.decommit), (hooks.purge ?: ohooks.purge), (hooks.split ?: ohooks.split), + (hooks.merge ?: ohooks.merge)}; + return mallctlbymib(oid.data(), oid.size(), nullptr, nullptr, const_cast(&nhooks), sizeof(nhooks)); +} +#endif + +template struct GetObjFxn; +template struct GetObjFxn; +template struct GetObjFxn; +template struct GetObjFxn; + +template struct SetObjFxn; +template struct SetObjFxn; +template struct SetObjFxn; +template struct SetObjFxn; + +const GetObjFxn thread_arena_hooks{"arena.0.chunk_hooks"}; +const SetObjFxn set_thread_arena_hooks{"arena.0.chunk_hooks"}; + +// request-or-sense new values in statistics +const GetObjFxn epoch{"epoch"}; + +// request separated page sets for each NUMA node (when created) +const GetObjFxn do_arenas_extend{"arenas.extend"}; // unsigned r- + +// assigned arena for local thread +const GetObjFxn thread_arena{"thread.arena"}; // unsigned rw +const SetObjFxn set_thread_arena{"thread.arena"}; // unsigned rw +const DoObjFxn do_thread_tcache_flush{"thread.tcache.flush"}; + +const GetObjFxn config_thp{"config.thp"}; +const GetObjFxn config_malloc_conf{"config.malloc_conf"}; + +const GetObjFxn thread_prof_name{"thread.prof.name"}; +const SetObjFxn set_thread_prof_name{"thread.prof.name"}; + +const GetObjFxn thread_prof_active{"thread.prof.active"}; +const SetObjFxn set_thread_prof_active{"thread.prof.active"}; + +int const proc_arena = 0; // default arena for jemalloc + +#if !HAVE_LIBJEMALLOC +int const proc_arena_nodump = 0; // default arena for jemalloc +#else +namespace +{ + chunk_alloc_t *s_origAllocHook = nullptr; // safe pre-main +} + +int const proc_arena_nodump = []() { + auto origArena = jemallctl::thread_arena(); + + int n = jemallctl::do_arenas_extend(); + jemallctl::set_thread_arena(n); + + chunk_hooks_t origHooks = jemallctl::thread_arena_hooks(); + s_origAllocHook = origHooks.alloc; + + origHooks.alloc = [](void *old, size_t len, size_t aligned, bool *zero, bool *commit, unsigned arena) { + void *r = (*s_origAllocHook)(old, len, aligned, zero, commit, arena); + + if (r) { + madvise(r, aligned_spacing(len, aligned), MADV_DONTDUMP); + } + + return r; + }; + + jemallctl::set_thread_arena_hooks(origHooks); + jemallctl::set_thread_arena(origArena); // default again + return n; +}(); +#endif + +} // namespace jemallctl diff --git a/lib/ts/jemallctl.h b/lib/ts/jemallctl.h new file mode 100644 index 00000000000..0ac4abc6fd2 --- /dev/null +++ b/lib/ts/jemallctl.h @@ -0,0 +1,96 @@ +/** @file + + Memory allocation routines for libts. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#ifndef _JEMALLCTL_H +#define _JEMALLCTL_H + +#include "ts/ink_platform.h" +#include "ts/ink_memory.h" + +#include + +#if !HAVE_LIBJEMALLOC +struct chunk_hooks_t { +}; +#endif + +namespace jemallctl +{ +extern int const proc_arena; +extern int const proc_arena_nodump; + +using objpath_t = std::vector; + +struct ObjBase { + ObjBase(const char *name); + +protected: + const objpath_t _oid; +}; + +template struct GetObjFxn : public ObjBase { + using ObjBase::ObjBase; + auto operator()(void) const -> T_VALUE; +}; + +template struct SetObjFxn : public ObjBase { + using ObjBase::ObjBase; + auto operator()(const T_VALUE &) const -> int; +}; + +template <> struct GetObjFxn : public ObjBase { + using ObjBase::ObjBase; + int operator()(void) const; +}; + +using DoObjFxn = GetObjFxn; + +chunk_hooks_t const &get_hugepage_hooks(); +chunk_hooks_t const &get_hugepage_nodump_hooks(); + +extern const GetObjFxn thread_arena_hooks; +extern const SetObjFxn set_thread_arena_hooks; + +// request-or-sense new values in statistics +extern const GetObjFxn epoch; + +// request separated page sets for each NUMA node (when created) +extern const GetObjFxn do_arenas_extend; + +// assigned arena for local thread +extern const GetObjFxn thread_arena; +extern const SetObjFxn set_thread_arena; +extern const DoObjFxn do_thread_tcache_flush; + +// from the build-time config +extern const GetObjFxn config_thp; +extern const GetObjFxn config_malloc_conf; + +// for profiling only +extern const GetObjFxn thread_prof_name; +extern const SetObjFxn set_thread_prof_name; +extern const GetObjFxn thread_prof_active; +extern const SetObjFxn set_thread_prof_active; +} + +#endif // _JEMALLCTL_H diff --git a/lib/ts/test_X509HostnameValidator.cc b/lib/ts/test_X509HostnameValidator.cc index 674b30e359a..39f6fb9be0b 100644 --- a/lib/ts/test_X509HostnameValidator.cc +++ b/lib/ts/test_X509HostnameValidator.cc @@ -28,6 +28,7 @@ #include "ts/ink_apidefs.h" #include "ts/Diags.h" +#include "ts/Allocator.h" #include "ts/ink_resource.h" #include "ts/ink_queue.h" #include "ts/X509HostnameValidator.h" diff --git a/lib/ts/test_freelist.cc b/lib/ts/test_freelist.cc index 19d269b514c..e05568bbd77 100644 --- a/lib/ts/test_freelist.cc +++ b/lib/ts/test_freelist.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include "ts/Allocator.h" + #include #include #include "ts/ink_thread.h" diff --git a/proxy/logging/LogUtils.h b/proxy/logging/LogUtils.h index b72e97da948..8f5a7ec5ec0 100644 --- a/proxy/logging/LogUtils.h +++ b/proxy/logging/LogUtils.h @@ -28,6 +28,8 @@ #include "ts/ink_platform.h" #include "ts/Arena.h" +#include "ts/apidefs.h" + namespace LogUtils { enum AlarmType {