diff --git a/3rdparty/snmalloc/CMakeLists.txt b/3rdparty/snmalloc/CMakeLists.txt index 611d7302e896..177464f55042 100644 --- a/3rdparty/snmalloc/CMakeLists.txt +++ b/3rdparty/snmalloc/CMakeLists.txt @@ -71,6 +71,9 @@ macro(clangformat_targets) else () message(STATUS "Generating clangformat target using ${CLANG_FORMAT}") file(GLOB_RECURSE ALL_SOURCE_FILES *.cc *.h *.hh) + # clangformat does not yet understand concepts well; for the moment, don't + # ask it to format them. See https://reviews.llvm.org/D79773 + list(FILTER ALL_SOURCE_FILES EXCLUDE REGEX "src/pal/pal_concept\.h$") add_custom_target( clangformat COMMAND ${CLANG_FORMAT} @@ -169,13 +172,14 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /DEBUG") else() add_compile_options(-fno-exceptions -fno-rtti -g -fomit-frame-pointer) - # Static TLS model unsupported on Haiku + # Static TLS model is unsupported on Haiku. + # All symbols are always dynamic on haiku and -rdynamic is redundant (and unsupported). if (NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") add_compile_options(-ftls-model=initial-exec) - endif() - if(SNMALLOC_CI_BUILD OR (${CMAKE_BUILD_TYPE} MATCHES "Debug")) - # Get better stack traces in CI and Debug. - target_link_libraries(snmalloc_lib INTERFACE "-rdynamic") + if(SNMALLOC_CI_BUILD OR (${CMAKE_BUILD_TYPE} MATCHES "Debug")) + # Get better stack traces in CI and Debug. + target_link_libraries(snmalloc_lib INTERFACE "-rdynamic") + endif() endif() if(SNMALLOC_OPTIMISE_FOR_CURRENT_MACHINE) @@ -255,7 +259,7 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) add_shim(snmallocshim SHARED ${SHARED_FILES}) add_shim(snmallocshim-1mib SHARED ${SHARED_FILES}) add_shim(snmallocshim-16mib SHARED ${SHARED_FILES}) - target_compile_definitions(snmallocshim-16mib PRIVATE SNMALOC_USE_LARGE_CHUNKS) + target_compile_definitions(snmallocshim-16mib PRIVATE SNMALLOC_USE_LARGE_CHUNKS) # Build a shim with some settings from oe. add_shim(snmallocshim-oe SHARED ${SHARED_FILES}) oe_simulate(snmallocshim-oe) @@ -276,7 +280,18 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) foreach(TEST_CATEGORY ${TEST_CATEGORIES}) subdirlist(TESTS ${TESTDIR}/${TEST_CATEGORY}) foreach(TEST ${TESTS}) - foreach(SUPER_SLAB_SIZE 1;16;oe) + if (WIN32 + OR (CMAKE_SYSTEM_NAME STREQUAL NetBSD) + OR (CMAKE_SYSTEM_NAME STREQUAL OpenBSD) + OR (CMAKE_SYSTEM_NAME STREQUAL DragonFly)) + # Windows does not support aligned allocation well enough + # for pass through. + # NetBSD, OpenBSD and DragonFlyBSD do not support malloc*size calls. + set(FLAVOURS 1;16;oe) + else() + set(FLAVOURS 1;16;oe;malloc) + endif() + foreach(SUPER_SLAB_SIZE ${FLAVOURS}) unset(SRC) aux_source_directory(${TESTDIR}/${TEST_CATEGORY}/${TEST} SRC) set(TESTNAME "${TEST_CATEGORY}-${TEST}-${SUPER_SLAB_SIZE}") @@ -285,9 +300,12 @@ if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY) if (${SUPER_SLAB_SIZE} EQUAL 16) target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_USE_LARGE_CHUNKS) endif() - if (${SUPER_SLAB_SIZE} EQUAL oe) + if (${SUPER_SLAB_SIZE} STREQUAL "oe") oe_simulate(${TESTNAME}) endif() + if (${SUPER_SLAB_SIZE} STREQUAL "malloc") + target_compile_definitions(${TESTNAME} PRIVATE SNMALLOC_PASS_THROUGH) + endif() target_link_libraries(${TESTNAME} snmalloc_lib) if (${TEST} MATCHES "release-.*") message(STATUS "Adding test: ${TESTNAME} only for release configs") diff --git a/3rdparty/snmalloc/README.md b/3rdparty/snmalloc/README.md index 5f69911f2f7c..84a0db27e27b 100644 --- a/3rdparty/snmalloc/README.md +++ b/3rdparty/snmalloc/README.md @@ -160,12 +160,12 @@ your system. The PAL must implement the following methods: ```c++ -[[noreturn]] void error(const char* const str) noexcept; +[[noreturn]] static void error(const char* const str) noexcept; ``` Report a fatal error and exit. ```c++ -void notify_not_using(void* p, size_t size) noexcept; +static void notify_not_using(void* p, size_t size) noexcept; ``` Notify the system that the range of memory from `p` to `p` + `size` is no longer in use, allowing the underlying physical pages to recycled for other @@ -173,7 +173,7 @@ purposes. ```c++ template -void notify_using(void* p, size_t size) noexcept; +static void notify_using(void* p, size_t size) noexcept; ``` Notify the system that the range of memory from `p` to `p` + `size` is now in use. On systems that lazily provide physical memory to virtual mappings, this @@ -183,7 +183,7 @@ responsible for ensuring that the newly requested memory is full of zeros. ```c++ template -void zero(void* p, size_t size) noexcept; +static void zero(void* p, size_t size) noexcept; ``` Zero the range of memory from `p` to `p` + `size`. This may be a simple `memset` call, but the `page_aligned` template parameter @@ -194,8 +194,8 @@ pages, rather than zeroing them synchronously in this call ```c++ template -void* reserve_aligned(size_t size) noexcept; -std::pair reserve_at_least(size_t size) noexcept; +static void* reserve_aligned(size_t size) noexcept; +static std::pair reserve_at_least(size_t size) noexcept; ``` Only one of these needs to be implemented, depending on whether the underlying system can provide strongly aligned memory regions. diff --git a/3rdparty/snmalloc/src/ds/aba.h b/3rdparty/snmalloc/src/ds/aba.h index fc8e3f8f7123..5d4e092b70a1 100644 --- a/3rdparty/snmalloc/src/ds/aba.h +++ b/3rdparty/snmalloc/src/ds/aba.h @@ -85,6 +85,13 @@ namespace snmalloc Linked old; ABA* parent; + /* + * MSVC apparently does not like the implicit constructor it creates when + * asked to interpret its input as C++20; it rejects the construction up + * in read(), above. Help it out by making the constructor explicit. + */ + Cmp(Linked old, ABA* parent) : old(old), parent(parent) {} + T* ptr() { return old.ptr; diff --git a/3rdparty/snmalloc/src/ds/bits.h b/3rdparty/snmalloc/src/ds/bits.h index 50967fd37aac..b59773c4353d 100644 --- a/3rdparty/snmalloc/src/ds/bits.h +++ b/3rdparty/snmalloc/src/ds/bits.h @@ -12,9 +12,6 @@ #include #include #include -#if defined(_WIN32) && defined(__GNUC__) -# define USE_CLZLL -#endif #ifdef pause # undef pause #endif @@ -76,10 +73,19 @@ namespace snmalloc return BITS - index - 1; # endif -#elif defined(USE_CLZLL) - return static_cast(__builtin_clzll(x)); #else - return static_cast(__builtin_clzl(x)); + if constexpr (std::is_same_v) + { + return static_cast(__builtin_clzl(x)); + } + else if constexpr (std::is_same_v) + { + return static_cast(__builtin_clzll(x)); + } + else if constexpr (std::is_same_v) + { + return static_cast(__builtin_clz(x)); + } #endif } @@ -142,17 +148,25 @@ namespace snmalloc inline size_t ctz(size_t x) { -#if __has_builtin(__builtin_ctzl) - return static_cast(__builtin_ctzl(x)); -#elif defined(_MSC_VER) +#if defined(_MSC_VER) # ifdef SNMALLOC_VA_BITS_64 return _tzcnt_u64(x); # else return _tzcnt_u32((uint32_t)x); # endif #else - // Probably GCC at this point. - return static_cast(__builtin_ctzl(x)); + if constexpr (std::is_same_v) + { + return static_cast(__builtin_ctzl(x)); + } + else if constexpr (std::is_same_v) + { + return static_cast(__builtin_ctzll(x)); + } + else if constexpr (std::is_same_v) + { + return static_cast(__builtin_ctz(x)); + } #endif } @@ -227,18 +241,19 @@ namespace snmalloc return BITS - clz_const(x - 1); } - static SNMALLOC_FAST_PATH size_t align_down(size_t value, size_t alignment) + constexpr SNMALLOC_FAST_PATH size_t + align_down(size_t value, size_t alignment) { - SNMALLOC_ASSERT(next_pow2(alignment) == alignment); + SNMALLOC_ASSERT(next_pow2_const(alignment) == alignment); size_t align_1 = alignment - 1; value &= ~align_1; return value; } - static inline size_t align_up(size_t value, size_t alignment) + constexpr SNMALLOC_FAST_PATH size_t align_up(size_t value, size_t alignment) { - SNMALLOC_ASSERT(next_pow2(alignment) == alignment); + SNMALLOC_ASSERT(next_pow2_const(alignment) == alignment); size_t align_1 = alignment - 1; value += align_1; diff --git a/3rdparty/snmalloc/src/ds/concept.h b/3rdparty/snmalloc/src/ds/concept.h new file mode 100644 index 000000000000..d935417d9b51 --- /dev/null +++ b/3rdparty/snmalloc/src/ds/concept.h @@ -0,0 +1,40 @@ +#pragma once + +/** + * C++20 concepts are referenced as if they were types in declarations within + * template parameters (e.g. "template ..."). That is, they + * take the place of the "typename"/"class" keyword on template parameters. + * If the compiler understands concepts, this macro expands as its argument; + * otherwise, it expands to the keyword "typename", so snmalloc templates that + * use concept-qualified parameters should use this to remain compatible across + * C++ versions: "template" + */ +#ifdef __cpp_concepts +# define SNMALLOC_CONCEPT(c) c +#else +# define SNMALLOC_CONCEPT(c) typename +#endif + +#ifdef __cpp_concepts +namespace snmalloc +{ + /** + * C++20 concepts are more than just new syntax; there's a new support + * library specified as well. As C++20 is quite new, however, there are some + * environments, notably Clang, that understand the syntax but do not yet + * offer the library. Fortunately, alternate pronouciations are possible. + */ +# ifdef _cpp_lib_concepts + /** + * ConceptSame is true if T and U are the same type and false otherwise. + * When specifying a concept, use ConceptSame to indicate that an + * expression must evaluate precisely to the type U. + */ + template + concept ConceptSame = std::same_as; +# else + template + concept ConceptSame = std::is_same::value; +# endif +} // namespace snmalloc +#endif diff --git a/3rdparty/snmalloc/src/ds/defines.h b/3rdparty/snmalloc/src/ds/defines.h index 7b9735912d4a..7ccbdb27224e 100644 --- a/3rdparty/snmalloc/src/ds/defines.h +++ b/3rdparty/snmalloc/src/ds/defines.h @@ -57,6 +57,15 @@ namespace snmalloc } #endif +#define SNMALLOC_CHECK(expr) \ + { \ + if (!(expr)) \ + { \ + snmalloc::error("Check fail: " #expr " in " __FILE__ \ + " on " TOSTRING(__LINE__)); \ + } \ + } + #ifndef NDEBUG # define SNMALLOC_ASSUME(x) SNMALLOC_ASSERT(x) #else diff --git a/3rdparty/snmalloc/src/mem/address_space.h b/3rdparty/snmalloc/src/mem/address_space.h index 2b41ac7b8c0c..31b436d38b0e 100644 --- a/3rdparty/snmalloc/src/mem/address_space.h +++ b/3rdparty/snmalloc/src/mem/address_space.h @@ -13,8 +13,8 @@ namespace snmalloc * It cannot unreserve memory, so this does not require the * usual complexity of a buddy allocator. */ - template - class AddressSpaceManager : public Pal + template + class AddressSpaceManager { /** * Stores the blocks of address space @@ -160,7 +160,7 @@ namespace snmalloc auto page_start = pointer_align_down(base); auto page_end = pointer_align_up(pointer_offset(base, size)); - Pal::template notify_using( + PAL::template notify_using( page_start, static_cast(page_end - page_start)); } @@ -178,11 +178,10 @@ namespace snmalloc SNMALLOC_ASSERT(bits::next_pow2(size) == size); SNMALLOC_ASSERT(size >= sizeof(void*)); - if constexpr (pal_supports) + if constexpr (pal_supports) { - if (size >= Pal::minimum_alloc_size) - return static_cast(this)->template reserve_aligned( - size); + if (size >= PAL::minimum_alloc_size) + return PAL::template reserve_aligned(size); } void* res; @@ -194,20 +193,18 @@ namespace snmalloc // Allocation failed ask OS for more memory void* block; size_t block_size; - if constexpr (pal_supports) + if constexpr (pal_supports) { - block_size = Pal::minimum_alloc_size; - block = static_cast(this)->template reserve_aligned( - block_size); + block_size = PAL::minimum_alloc_size; + block = PAL::template reserve_aligned(block_size); } else { // Need at least 2 times the space to guarantee alignment. // Hold lock here as a race could cause additional requests to - // the Pal, and this could lead to suprious OOM. This is - // particularly bad if the Pal gives all the memory on first call. - auto block_and_size = - static_cast(this)->reserve_at_least(size * 2); + // the PAL, and this could lead to suprious OOM. This is + // particularly bad if the PAL gives all the memory on first call. + auto block_and_size = PAL::reserve_at_least(size * 2); block = block_and_size.first; block_size = block_and_size.second; diff --git a/3rdparty/snmalloc/src/mem/alloc.h b/3rdparty/snmalloc/src/mem/alloc.h index 74380d1d7854..26b3ed76a230 100644 --- a/3rdparty/snmalloc/src/mem/alloc.h +++ b/3rdparty/snmalloc/src/mem/alloc.h @@ -10,6 +10,7 @@ #include "../test/histogram.h" #include "allocstats.h" #include "chunkmap.h" +#include "external_alloc.h" #include "largealloc.h" #include "mediumslab.h" #include "pooled.h" @@ -125,14 +126,18 @@ namespace snmalloc SNMALLOC_FAST_PATH ALLOCATOR void* alloc() { static_assert(size != 0, "Size must not be zero."); -#ifdef USE_MALLOC +#ifdef SNMALLOC_PASS_THROUGH static_assert( allow_reserve == YesReserve, "When passing to malloc, cannot require NoResereve"); - if constexpr (zero_mem == NoZero) - return malloc(size); - else - return calloc(1, size); + // snmalloc guarantees a lot of alignment, so we can depend on this + // make pass through call aligned_alloc with the alignment snmalloc + // would guarantee. + void* result = external_alloc::aligned_alloc( + natural_alignment(size), round_size(size)); + if constexpr (zero_mem == YesZero) + memset(result, 0, size); + return result; #else constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); @@ -162,14 +167,18 @@ namespace snmalloc template SNMALLOC_FAST_PATH ALLOCATOR void* alloc(size_t size) { -#ifdef USE_MALLOC +#ifdef SNMALLOC_PASS_THROUGH static_assert( allow_reserve == YesReserve, "When passing to malloc, cannot require NoResereve"); - if constexpr (zero_mem == NoZero) - return malloc(size); - else - return calloc(1, size); + // snmalloc guarantees a lot of alignment, so we can depend on this + // make pass through call aligned_alloc with the alignment snmalloc + // would guarantee. + void* result = external_alloc::aligned_alloc( + natural_alignment(size), round_size(size)); + if constexpr (zero_mem == YesZero) + memset(result, 0, size); + return result; #else // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. @@ -241,9 +250,9 @@ namespace snmalloc template void dealloc(void* p) { -#ifdef USE_MALLOC +#ifdef SNMALLOC_PASS_THROUGH UNUSED(size); - return free(p); + return external_alloc::free(p); #else check_size(p, size); constexpr sizeclass_t sizeclass = size_to_sizeclass_const(size); @@ -281,9 +290,9 @@ namespace snmalloc */ SNMALLOC_FAST_PATH void dealloc(void* p, size_t size) { -#ifdef USE_MALLOC +#ifdef SNMALLOC_PASS_THROUGH UNUSED(size); - return free(p); + return external_alloc::free(p); #else SNMALLOC_ASSERT(p != nullptr); check_size(p, size); @@ -327,8 +336,8 @@ namespace snmalloc */ SNMALLOC_FAST_PATH void dealloc(void* p) { -#ifdef USE_MALLOC - return free(p); +#ifdef SNMALLOC_PASS_THROUGH + return external_alloc::free(p); #else uint8_t size = chunkmap().get(address_cast(p)); @@ -397,7 +406,7 @@ namespace snmalloc template static void* external_pointer(void* p) { -#ifdef USE_MALLOC +#ifdef SNMALLOC_PASS_THROUGH error("Unsupported"); UNUSED(p); #else @@ -463,6 +472,9 @@ namespace snmalloc public: SNMALLOC_FAST_PATH static size_t alloc_size(const void* p) { +#ifdef SNMALLOC_PASS_THROUGH + return external_alloc::malloc_usable_size(const_cast(p)); +#else // This must be called on an external pointer. size_t size = ChunkMap::get(address_cast(p)); @@ -492,6 +504,7 @@ namespace snmalloc } return alloc_size_error(); +#endif } size_t get_id() @@ -1063,7 +1076,7 @@ namespace snmalloc void* p = remove_cache_friendly_offset(head, sizeclass); if constexpr (zero_mem == YesZero) { - large_allocator.memory_provider.zero(p, sizeclass_to_size(sizeclass)); + MemoryProvider::Pal::zero(p, sizeclass_to_size(sizeclass)); } return p; } @@ -1109,8 +1122,8 @@ namespace snmalloc SlabLink* link = sl.get_next(); slab = get_slab(link); auto& ffl = small_fast_free_lists[sizeclass]; - return slab->alloc( - sl, ffl, rsize, large_allocator.memory_provider); + return slab->alloc( + sl, ffl, rsize); } return small_alloc_rare(sizeclass, size); } @@ -1182,7 +1195,7 @@ namespace snmalloc if constexpr (zero_mem == YesZero) { - large_allocator.memory_provider.zero(p, sizeclass_to_size(sizeclass)); + MemoryProvider::Pal::zero(p, sizeclass_to_size(sizeclass)); } return p; } @@ -1313,7 +1326,7 @@ namespace snmalloc if (slab != nullptr) { - p = slab->alloc(size, large_allocator.memory_provider); + p = slab->alloc(size); if (slab->full()) sc->pop(); @@ -1336,7 +1349,7 @@ namespace snmalloc slab->init(public_state(), sizeclass, rsize); chunkmap().set_slab(slab); - p = slab->alloc(size, large_allocator.memory_provider); + p = slab->alloc(size); if (!slab->full()) sc->insert(slab); diff --git a/3rdparty/snmalloc/src/mem/external_alloc.h b/3rdparty/snmalloc/src/mem/external_alloc.h new file mode 100644 index 000000000000..ae35870fdfd7 --- /dev/null +++ b/3rdparty/snmalloc/src/mem/external_alloc.h @@ -0,0 +1,66 @@ +#pragma once + +#ifdef SNMALLOC_PASS_THROUGH +# if defined(__HAIKU__) +# define _GNU_SOURCE +# endif +# include +# if defined(_WIN32) //|| defined(__APPLE__) +# error "Pass through not supported on this platform" +// The Windows aligned allocation API is not capable of supporting the +// snmalloc API Apple was not providing aligned memory in some tests. +# else +// Defines malloc_size for the platform. +# if defined(_WIN32) +namespace snmalloc::external_alloc +{ + inline size_t malloc_usable_size(void* ptr) + { + return _msize(ptr); + } +} +# elif defined(__APPLE__) +# include +namespace snmalloc::external_alloc +{ + inline size_t malloc_usable_size(void* ptr) + { + return malloc_size(ptr); + } +} +# elif defined(__linux__) || defined(__HAIKU__) +# include +namespace snmalloc::external_alloc +{ + using ::malloc_usable_size; +} +# elif defined(__sun) || defined(__NetBSD__) || defined(__OpenBSD__) +namespace snmalloc::external_alloc +{ + using ::malloc_usable_size; +} +# elif defined(__FreeBSD__) +# include +namespace snmalloc::external_alloc +{ + using ::malloc_usable_size; +} +# else +# error Define malloc size macro for this platform. +# endif +namespace snmalloc::external_alloc +{ + inline void* aligned_alloc(size_t alignment, size_t size) + { + void* result; + if (posix_memalign(&result, alignment, size) != 0) + { + result = nullptr; + } + return result; + } + + using ::free; +} +# endif +#endif diff --git a/3rdparty/snmalloc/src/mem/globalalloc.h b/3rdparty/snmalloc/src/mem/globalalloc.h index dac401f82a97..f94ad45ad9e7 100644 --- a/3rdparty/snmalloc/src/mem/globalalloc.h +++ b/3rdparty/snmalloc/src/mem/globalalloc.h @@ -72,7 +72,7 @@ namespace snmalloc void cleanup_unused() { -#ifndef USE_MALLOC +#ifndef SNMALLOC_PASS_THROUGH // Call this periodically to free and coalesce memory allocated by // allocators that are not currently in use by any thread. // One atomic operation to extract the stack, another to restore it. @@ -102,7 +102,7 @@ namespace snmalloc */ void debug_check_empty(bool* result = nullptr) { -#ifndef USE_MALLOC +#ifndef SNMALLOC_PASS_THROUGH // This is a debugging function. It checks that all memory from all // allocators has been freed. auto* alloc = Parent::iterate(); diff --git a/3rdparty/snmalloc/src/mem/largealloc.h b/3rdparty/snmalloc/src/mem/largealloc.h index 114fa3918ae7..34f5ce7e7af7 100644 --- a/3rdparty/snmalloc/src/mem/largealloc.h +++ b/3rdparty/snmalloc/src/mem/largealloc.h @@ -14,7 +14,7 @@ namespace snmalloc { - template + template class MemoryProviderStateMixin; class Largeslab : public Baseslab @@ -24,7 +24,7 @@ namespace snmalloc private: template friend class MPMCStack; - template + template friend class MemoryProviderStateMixin; std::atomic next; @@ -56,8 +56,8 @@ namespace snmalloc // This represents the state that the large allcoator needs to add to the // global state of the allocator. This is currently stored in the memory // provider, so we add this in. - template - class MemoryProviderStateMixin : public PalNotificationObject, public PAL + template + class MemoryProviderStateMixin : public PalNotificationObject { /** * Simple flag for checking if another instance of lazy-decommit is @@ -70,7 +70,19 @@ namespace snmalloc */ AddressSpaceManager address_space = {}; + /** + * High-water mark of used memory. + */ + std::atomic peak_memory_used_bytes{0}; + public: + using Pal = PAL; + + /** + * Memory current available in large_stacks + */ + std::atomic available_large_chunks_in_bytes{0}; + /** * Stack of large allocations that have been returned for reuse. */ @@ -184,10 +196,13 @@ namespace snmalloc { // Cache line align size_t size = bits::align_up(sizeof(T), 64); - size = bits::max(size, alignment); - void* p = address_space.template reserve(bits::next_pow2(size)); + size = bits::next_pow2(bits::max(size, alignment)); + void* p = address_space.template reserve(size); if (p == nullptr) return nullptr; + + peak_memory_used_bytes += size; + return new (p) T(std::forward(args)...); } @@ -195,9 +210,20 @@ namespace snmalloc void* reserve(size_t large_class) noexcept { size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - + peak_memory_used_bytes += size; return address_space.template reserve(size); } + + /** + * Returns a pair of current memory usage and peak memory usage. + * Both statistics are very coarse-grained. + */ + std::pair memory_usage() + { + size_t avail = available_large_chunks_in_bytes; + size_t peak = peak_memory_used_bytes; + return {peak - avail, peak}; + } }; using Stats = AllocStats; @@ -234,11 +260,12 @@ namespace snmalloc p = memory_provider.template reserve(large_class); if (p == nullptr) return nullptr; - memory_provider.template notify_using(p, rsize); + MemoryProvider::Pal::template notify_using(p, rsize); } else { stats.superslab_pop(); + memory_provider.available_large_chunks_in_bytes -= rsize; // Cross-reference alloc.h's large_dealloc decommitment condition. bool decommitted = @@ -251,19 +278,19 @@ namespace snmalloc // The first page is already in "use" for the stack element, // this will need zeroing for a YesZero call. if constexpr (zero_mem == YesZero) - memory_provider.template zero(p, OS_PAGE_SIZE); + MemoryProvider::Pal::template zero(p, OS_PAGE_SIZE); // Notify we are using the rest of the allocation. // Passing zero_mem ensures the PAL provides zeroed pages if // required. - memory_provider.template notify_using( + MemoryProvider::Pal::template notify_using( pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE); } else { // This is a superslab that has not been decommitted. if constexpr (zero_mem == YesZero) - memory_provider.template zero( + MemoryProvider::Pal::template zero( p, bits::align_up(size, OS_PAGE_SIZE)); else UNUSED(size); @@ -279,23 +306,24 @@ namespace snmalloc if constexpr (decommit_strategy == DecommitSuperLazy) { static_assert( - pal_supports, + pal_supports, "A lazy decommit strategy cannot be implemented on platforms " "without low memory notifications"); } + size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; + // Cross-reference largealloc's alloc() decommitted condition. if ( (decommit_strategy != DecommitNone) && (large_class != 0 || decommit_strategy == DecommitSuper)) { - size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class; - - memory_provider.notify_not_using( + MemoryProvider::Pal::notify_not_using( pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE); } stats.superslab_push(); + memory_provider.available_large_chunks_in_bytes += rsize; memory_provider.large_stack[large_class].push(static_cast(p)); } }; diff --git a/3rdparty/snmalloc/src/mem/mediumslab.h b/3rdparty/snmalloc/src/mem/mediumslab.h index 59a10f2e0238..9dcfa665125f 100644 --- a/3rdparty/snmalloc/src/mem/mediumslab.h +++ b/3rdparty/snmalloc/src/mem/mediumslab.h @@ -76,8 +76,8 @@ namespace snmalloc return sizeclass; } - template - void* alloc(size_t size, MemoryProvider& memory_provider) + template + void* alloc(size_t size) { SNMALLOC_ASSERT(!full()); @@ -86,7 +86,7 @@ namespace snmalloc free--; if constexpr (zero_mem == YesZero) - memory_provider.zero(p, size); + PAL::zero(p, size); else UNUSED(size); diff --git a/3rdparty/snmalloc/src/mem/pagemap.h b/3rdparty/snmalloc/src/mem/pagemap.h index bc75fd39d9ad..5b785723fbca 100644 --- a/3rdparty/snmalloc/src/mem/pagemap.h +++ b/3rdparty/snmalloc/src/mem/pagemap.h @@ -184,25 +184,35 @@ namespace snmalloc size_t shift = TOPLEVEL_SHIFT; std::atomic* e = &top[ix]; - for (size_t i = 0; i < INDEX_LEVELS; i++) + // This is effectively a + // for (size_t i = 0; i < INDEX_LEVELS; i++) + // loop, but uses constexpr to guarantee optimised version + // where the INDEX_LEVELS in {0,1}. + if constexpr (INDEX_LEVELS != 0) { - PagemapEntry* value = get_node(e, result); - if (unlikely(!result)) - return {nullptr, 0}; + size_t i = 0; + while (true) + { + PagemapEntry* value = get_node(e, result); + if (unlikely(!result)) + return {nullptr, 0}; - shift -= BITS_PER_INDEX_LEVEL; - ix = (static_cast(addr) >> shift) & ENTRIES_MASK; - e = &value->entries[ix]; + shift -= BITS_PER_INDEX_LEVEL; + ix = (static_cast(addr) >> shift) & ENTRIES_MASK; + e = &value->entries[ix]; - if constexpr (INDEX_LEVELS == 1) - { - UNUSED(i); - break; + if constexpr (INDEX_LEVELS == 1) + { + UNUSED(i); + break; + } + else + { + i++; + if (i == INDEX_LEVELS) + break; + } } - i++; - - if (i == INDEX_LEVELS) - break; } Leaf* leaf = reinterpret_cast(get_node(e, result)); @@ -326,9 +336,7 @@ namespace snmalloc private: static constexpr size_t COVERED_BITS = bits::ADDRESS_BITS - GRANULARITY_BITS; - static constexpr size_t CONTENT_BITS = - bits::next_pow2_bits_const(sizeof(T)); - static constexpr size_t ENTRIES = 1ULL << (COVERED_BITS + CONTENT_BITS); + static constexpr size_t ENTRIES = 1ULL << COVERED_BITS; static constexpr size_t SHIFT = GRANULARITY_BITS; std::atomic top[ENTRIES]; diff --git a/3rdparty/snmalloc/src/mem/pool.h b/3rdparty/snmalloc/src/mem/pool.h index c460567ed38b..69de2ca7a0f0 100644 --- a/3rdparty/snmalloc/src/mem/pool.h +++ b/3rdparty/snmalloc/src/mem/pool.h @@ -2,6 +2,7 @@ #include "../ds/flaglock.h" #include "../ds/mpmcstack.h" +#include "../pal/pal_concept.h" #include "pooled.h" namespace snmalloc @@ -21,7 +22,7 @@ namespace snmalloc { private: friend Pooled; - template + template friend class MemoryProviderStateMixin; std::atomic_flag lock = ATOMIC_FLAG_INIT; diff --git a/3rdparty/snmalloc/src/mem/sizeclass.h b/3rdparty/snmalloc/src/mem/sizeclass.h index 801192d830b9..5321aaf28d5d 100644 --- a/3rdparty/snmalloc/src/mem/sizeclass.h +++ b/3rdparty/snmalloc/src/mem/sizeclass.h @@ -198,4 +198,12 @@ namespace snmalloc } return sizeclass_to_size(size_to_sizeclass(size)); } + + /// Returns the alignment that this size naturally has, that is + /// all allocations of size `size` will be aligned to the returned value. + SNMALLOC_FAST_PATH static size_t natural_alignment(size_t size) + { + auto rsize = round_size(size); + return bits::one_at_bit(bits::ctz(rsize)); + } } // namespace snmalloc diff --git a/3rdparty/snmalloc/src/mem/slab.h b/3rdparty/snmalloc/src/mem/slab.h index 8c14fe364422..c59e82e62bbf 100644 --- a/3rdparty/snmalloc/src/mem/slab.h +++ b/3rdparty/snmalloc/src/mem/slab.h @@ -36,12 +36,9 @@ namespace snmalloc * Returns the link as the allocation, and places the free list into the * `fast_free_list` for further allocations. */ - template - SNMALLOC_FAST_PATH void* alloc( - SlabList& sl, - FreeListHead& fast_free_list, - size_t rsize, - MemoryProvider& memory_provider) + template + SNMALLOC_FAST_PATH void* + alloc(SlabList& sl, FreeListHead& fast_free_list, size_t rsize) { // Read the head from the metadata stored in the superslab. Metaslab& meta = get_meta(); @@ -73,9 +70,9 @@ namespace snmalloc if constexpr (zero_mem == YesZero) { if (rsize < PAGE_ALIGNED_SIZE) - memory_provider.zero(p, rsize); + PAL::zero(p, rsize); else - memory_provider.template zero(p, rsize); + PAL::template zero(p, rsize); } else { diff --git a/3rdparty/snmalloc/src/mem/superslab.h b/3rdparty/snmalloc/src/mem/superslab.h index 50e6e1630403..93aabb908ce1 100644 --- a/3rdparty/snmalloc/src/mem/superslab.h +++ b/3rdparty/snmalloc/src/mem/superslab.h @@ -81,7 +81,10 @@ namespace snmalloc { allocator = alloc; - if (kind != Super) + // If Superslab is larger than a page, then we cannot guarantee it still + // has a valid layout as the subsequent pages could have been freed and + // zeroed, hence only skip initialisation if smaller. + if (kind != Super || (sizeof(Superslab) >= OS_PAGE_SIZE)) { if (kind != Fresh) { diff --git a/3rdparty/snmalloc/src/mem/threadalloc.h b/3rdparty/snmalloc/src/mem/threadalloc.h index ba8d2fa3efe7..306c78215e69 100644 --- a/3rdparty/snmalloc/src/mem/threadalloc.h +++ b/3rdparty/snmalloc/src/mem/threadalloc.h @@ -189,7 +189,7 @@ namespace snmalloc */ static SNMALLOC_FAST_PATH Alloc* get() { -# ifdef USE_MALLOC +# ifdef SNMALLOC_PASS_THROUGH return get_reference(); # else auto*& alloc = get_reference(); diff --git a/3rdparty/snmalloc/src/override/malloc-extensions.cc b/3rdparty/snmalloc/src/override/malloc-extensions.cc new file mode 100644 index 000000000000..ff621a98e9d2 --- /dev/null +++ b/3rdparty/snmalloc/src/override/malloc-extensions.cc @@ -0,0 +1,12 @@ +#include "malloc-extensions.h" + +#include "../snmalloc.h" + +using namespace snmalloc; + +void get_malloc_info_v1(malloc_info_v1* stats) +{ + auto next_memory_usage = default_memory_provider().memory_usage(); + stats->current_memory_usage = next_memory_usage.first; + stats->peak_memory_usage = next_memory_usage.second; +} \ No newline at end of file diff --git a/3rdparty/snmalloc/src/override/malloc-extensions.h b/3rdparty/snmalloc/src/override/malloc-extensions.h new file mode 100644 index 000000000000..2fc677743fe1 --- /dev/null +++ b/3rdparty/snmalloc/src/override/malloc-extensions.h @@ -0,0 +1,34 @@ +/** + * Malloc extensions + * + * This file contains additional non-standard API surface for snmalloc. + * The API is subject to changes, but will be clearly noted in release + * notes. + */ + +/** + * Structure for returning memory used by snmalloc. + * + * The statistics are very coarse grained as they only track + * usage at the superslab/chunk level. Meta-data and object + * data is not tracked independantly. + */ +struct malloc_info_v1 +{ + /** + * Current memory usage of the allocator. Extremely coarse + * grained for efficient calculation. + */ + size_t current_memory_usage; + + /** + * High-water mark of current_memory_usage. + */ + size_t peak_memory_usage; +}; + +/** + * Populates a malloc_info_v1 structure for the latest values + * from snmalloc. + */ +void get_malloc_info_v1(malloc_info_v1* stats); diff --git a/3rdparty/snmalloc/src/override/malloc.cc b/3rdparty/snmalloc/src/override/malloc.cc index fd2ff1c5e375..dc9f7a4ecae7 100644 --- a/3rdparty/snmalloc/src/override/malloc.cc +++ b/3rdparty/snmalloc/src/override/malloc.cc @@ -24,6 +24,18 @@ using namespace snmalloc; extern "C" { + void SNMALLOC_NAME_MANGLE(check_start)(void* ptr) + { +#if !defined(NDEBUG) && !defined(SNMALLOC_PASS_THROUGH) + if (Alloc::external_pointer(ptr) != ptr) + { + error("Using pointer that is not to the start of an allocation"); + } +#else + UNUSED(ptr); +#endif + } + SNMALLOC_EXPORT void* SNMALLOC_NAME_MANGLE(__malloc_end_pointer)(void* ptr) { return Alloc::external_pointer(ptr); @@ -36,6 +48,7 @@ extern "C" SNMALLOC_EXPORT void SNMALLOC_NAME_MANGLE(free)(void* ptr) { + SNMALLOC_NAME_MANGLE(check_start)(ptr); ThreadAlloc::get_noncachable()->dealloc(ptr); } @@ -79,24 +92,26 @@ extern "C" SNMALLOC_NAME_MANGLE(free)(ptr); return nullptr; } -#ifndef NDEBUG - // This check is redundant, because the check in memcpy will fail if this - // is skipped, but it's useful for debugging. - if (Alloc::external_pointer(ptr) != ptr) - { - error( - "Calling realloc on pointer that is not to the start of an allocation"); - } -#endif + + SNMALLOC_NAME_MANGLE(check_start)(ptr); + size_t sz = Alloc::alloc_size(ptr); // Keep the current allocation if the given size is in the same sizeclass. if (sz == round_size(size)) + { +#ifdef SNMALLOC_PASS_THROUGH + // snmallocs alignment guarantees can be broken by realloc in pass-through + // this is not exercised, by existing clients, but is tested. + if (pointer_align_up(ptr, natural_alignment(size)) == ptr) + return ptr; +#else return ptr; - +#endif + } void* p = SNMALLOC_NAME_MANGLE(malloc)(size); if (p != nullptr) { - SNMALLOC_ASSERT(p == Alloc::external_pointer(p)); + SNMALLOC_NAME_MANGLE(check_start)(p); sz = bits::min(size, sz); memcpy(p, ptr, sz); SNMALLOC_NAME_MANGLE(free)(ptr); diff --git a/3rdparty/snmalloc/src/pal/pal.h b/3rdparty/snmalloc/src/pal/pal.h index 47f30e0257fc..4ed353fd642c 100644 --- a/3rdparty/snmalloc/src/pal/pal.h +++ b/3rdparty/snmalloc/src/pal/pal.h @@ -1,5 +1,7 @@ #pragma once +#include "../ds/concept.h" +#include "pal_concept.h" #include "pal_consts.h" // If simultating OE, then we need the underlying platform @@ -8,6 +10,7 @@ #endif #if !defined(OPEN_ENCLAVE) || defined(OPEN_ENCLAVE_SIMULATION) # include "pal_apple.h" +# include "pal_dragonfly.h" # include "pal_freebsd.h" # include "pal_freebsd_kernel.h" # include "pal_haiku.h" @@ -41,6 +44,8 @@ namespace snmalloc PALOpenBSD; # elif defined(__sun) PALSolaris; +# elif defined(__DragonFly__) + PALDragonfly; # else # error Unsupported platform # endif @@ -63,7 +68,7 @@ namespace snmalloc /** * Query whether the PAL supports a specific feature. */ - template + template constexpr static bool pal_supports = (PAL::pal_features & F) == F; // Used to keep Superslab metadata committed. diff --git a/3rdparty/snmalloc/src/pal/pal_bsd.h b/3rdparty/snmalloc/src/pal/pal_bsd.h index c446dcbc1e53..c4086190e605 100644 --- a/3rdparty/snmalloc/src/pal/pal_bsd.h +++ b/3rdparty/snmalloc/src/pal/pal_bsd.h @@ -31,7 +31,7 @@ namespace snmalloc * operating system to replace the pages with CoW copies of a zero page at * any point between the call and the next write to that page. */ - void notify_not_using(void* p, size_t size) noexcept + static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); madvise(p, size, MADV_FREE); diff --git a/3rdparty/snmalloc/src/pal/pal_bsd_aligned.h b/3rdparty/snmalloc/src/pal/pal_bsd_aligned.h index d5d57661b7d5..7acefb71029a 100644 --- a/3rdparty/snmalloc/src/pal/pal_bsd_aligned.h +++ b/3rdparty/snmalloc/src/pal/pal_bsd_aligned.h @@ -29,7 +29,7 @@ namespace snmalloc * Reserve memory at a specific alignment. */ template - void* reserve_aligned(size_t size) noexcept + static void* reserve_aligned(size_t size) noexcept { // Alignment must be a power of 2. SNMALLOC_ASSERT(size == bits::next_pow2(size)); diff --git a/3rdparty/snmalloc/src/pal/pal_concept.h b/3rdparty/snmalloc/src/pal/pal_concept.h new file mode 100644 index 000000000000..0407c856ebde --- /dev/null +++ b/3rdparty/snmalloc/src/pal/pal_concept.h @@ -0,0 +1,101 @@ +#pragma once + +#ifdef __cpp_concepts +# include "../ds/concept.h" +# include "pal_consts.h" + +# include + +namespace snmalloc +{ + /** + * PALs must advertize the bit vector of their supported features and the + * platform's page size. This concept enforces that these are indeed + * constants that fit in the desired types. (This is subtly different from + * saying that they are the required types; C++ may handle constants without + * much regard for their claimed type.) + */ + template + concept ConceptPAL_static_members = requires() + { + typename std::integral_constant; + typename std::integral_constant; + }; + + /** + * PALs expose an error reporting function which takes a const C string. + */ + template + concept ConceptPAL_error = requires(const char* const str) + { + { PAL::error(str) } -> ConceptSame; + }; + + /** + * PALs expose a basic library of memory operations. + */ + template + concept ConceptPAL_memops = requires(void* vp, size_t sz) + { + { PAL::notify_not_using(vp, sz) } noexcept -> ConceptSame; + + { PAL::template notify_using(vp, sz) } noexcept + -> ConceptSame; + { PAL::template notify_using(vp, sz) } noexcept + -> ConceptSame; + + { PAL::template zero(vp, sz) } noexcept -> ConceptSame; + { PAL::template zero(vp, sz) } noexcept -> ConceptSame; + }; + + /** + * Absent any feature flags, the PAL must support a crude primitive allocator + */ + template + concept ConceptPAL_reserve_at_least = requires(PAL p, void* vp, size_t sz) + { + { PAL::reserve_at_least(sz) } noexcept + -> ConceptSame>; + }; + + /** + * Some PALs expose a richer allocator which understands aligned allocations + */ + template + concept ConceptPAL_reserve_aligned = requires(size_t sz) + { + { PAL::template reserve_aligned(sz) } noexcept -> ConceptSame; + { PAL::template reserve_aligned(sz) } noexcept + -> ConceptSame; + }; + + /** + * Some PALs can provide memory pressure callbacks. + */ + template + concept ConceptPAL_mem_low_notify = requires(PalNotificationObject* pno) + { + { PAL::expensive_low_memory_check() } -> ConceptSame; + { PAL::register_for_low_memory_callback(pno) } -> ConceptSame; + }; + + /** + * PALs ascribe to the conjunction of several concepts. These are broken + * out by the shape of the requires() quantifiers required and by any + * requisite claimed pal_features. PALs not claiming particular features + * are, naturally, not bound by the corresponding concept. + */ + template + concept ConceptPAL = + ConceptPAL_static_members && + ConceptPAL_error && + ConceptPAL_memops && + (!(PAL::pal_features & LowMemoryNotification) || + ConceptPAL_mem_low_notify) && + (!!(PAL::pal_features & AlignedAllocation) || + ConceptPAL_reserve_at_least) && + (!(PAL::pal_features & AlignedAllocation) || + ConceptPAL_reserve_aligned); + +} // namespace snmalloc +#endif diff --git a/3rdparty/snmalloc/src/pal/pal_dragonfly.h b/3rdparty/snmalloc/src/pal/pal_dragonfly.h new file mode 100644 index 000000000000..f0c3dafc3795 --- /dev/null +++ b/3rdparty/snmalloc/src/pal/pal_dragonfly.h @@ -0,0 +1,30 @@ +#pragma once + +#if defined(__DragonFly__) && !defined(_KERNEL) +# include "pal_bsd.h" + +namespace snmalloc +{ + /** + * DragonflyBSD-specific platform abstraction layer. + * + * This adds DragonFlyBSD-specific aligned allocation to the BSD + * implementation. + */ + class PALDragonfly : public PALBSD + { + public: + /** + * Bitmap of PalFeatures flags indicating the optional features that this + * PAL supports. + * + * The DragonflyBSD PAL does not currently add any features beyond + * of those of the BSD Pal. + * Like FreeBSD, MAP_NORESERVE is implicit. + * This field is declared explicitly to remind anyone modifying this class + * to add new features that they should add any required feature flags. + */ + static constexpr uint64_t pal_features = PALPOSIX::pal_features; + }; +} // namespace snmalloc +#endif diff --git a/3rdparty/snmalloc/src/pal/pal_freebsd_kernel.h b/3rdparty/snmalloc/src/pal/pal_freebsd_kernel.h index 826c1dc2a7c9..7bab485e8879 100644 --- a/3rdparty/snmalloc/src/pal/pal_freebsd_kernel.h +++ b/3rdparty/snmalloc/src/pal/pal_freebsd_kernel.h @@ -34,7 +34,7 @@ namespace snmalloc } /// Notify platform that we will not be using these pages - void notify_not_using(void* p, size_t size) + static void notify_not_using(void* p, size_t size) { vm_offset_t addr = get_vm_offset(p); kmem_unback(kernel_object, addr, size); @@ -42,7 +42,7 @@ namespace snmalloc /// Notify platform that we will be using these pages template - void notify_using(void* p, size_t size) + static void notify_using(void* p, size_t size) { vm_offset_t addr = get_vm_offset(p); int flags = M_WAITOK | ((zero_mem == YesZero) ? M_ZERO : 0); @@ -54,13 +54,13 @@ namespace snmalloc /// OS specific function for zeroing memory template - void zero(void* p, size_t size) + static void zero(void* p, size_t size) { ::bzero(p, size); } template - void* reserve_aligned(size_t size) noexcept + static void* reserve_aligned(size_t size) noexcept { SNMALLOC_ASSERT(size == bits::next_pow2(size)); SNMALLOC_ASSERT(size >= minimum_alloc_size); diff --git a/3rdparty/snmalloc/src/pal/pal_haiku.h b/3rdparty/snmalloc/src/pal/pal_haiku.h index 05ba71a06f45..1b7cc1e8afab 100644 --- a/3rdparty/snmalloc/src/pal/pal_haiku.h +++ b/3rdparty/snmalloc/src/pal/pal_haiku.h @@ -31,7 +31,7 @@ namespace snmalloc * Notify platform that we will not be needing these pages. * Haiku does not provide madvise call per say only the posix equivalent. */ - void notify_not_using(void* p, size_t size) noexcept + static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); posix_madvise(p, size, POSIX_MADV_DONTNEED); diff --git a/3rdparty/snmalloc/src/pal/pal_linux.h b/3rdparty/snmalloc/src/pal/pal_linux.h index a645c163ca10..74999afc3c40 100644 --- a/3rdparty/snmalloc/src/pal/pal_linux.h +++ b/3rdparty/snmalloc/src/pal/pal_linux.h @@ -37,7 +37,7 @@ namespace snmalloc * clear the underlying memory range. */ template - void zero(void* p, size_t size) noexcept + static void zero(void* p, size_t size) noexcept { // QEMU does not seem to be giving the desired behaviour for // MADV_DONTNEED. switch back to memset only for QEMU. diff --git a/3rdparty/snmalloc/src/pal/pal_open_enclave.h b/3rdparty/snmalloc/src/pal/pal_open_enclave.h index 0183c9f9cdef..420c49d573c5 100644 --- a/3rdparty/snmalloc/src/pal/pal_open_enclave.h +++ b/3rdparty/snmalloc/src/pal/pal_open_enclave.h @@ -62,7 +62,7 @@ namespace snmalloc } template - void zero(void* p, size_t size) noexcept + static void zero(void* p, size_t size) noexcept { oe_memset_s(p, size, 0, size); } diff --git a/3rdparty/snmalloc/src/pal/pal_plain.h b/3rdparty/snmalloc/src/pal/pal_plain.h index a7cd199907da..9d8c8d9ec2e2 100644 --- a/3rdparty/snmalloc/src/pal/pal_plain.h +++ b/3rdparty/snmalloc/src/pal/pal_plain.h @@ -11,11 +11,11 @@ namespace snmalloc { public: // Notify platform that we will not be using these pages - void notify_not_using(void*, size_t) noexcept {} + static void notify_not_using(void*, size_t) noexcept {} // Notify platform that we will not be using these pages template - void notify_using(void* p, size_t size) noexcept + static void notify_using(void* p, size_t size) noexcept { if constexpr (zero_mem == YesZero) { diff --git a/3rdparty/snmalloc/src/pal/pal_posix.h b/3rdparty/snmalloc/src/pal/pal_posix.h index 03d39ccce3ff..ba1f489a5c22 100644 --- a/3rdparty/snmalloc/src/pal/pal_posix.h +++ b/3rdparty/snmalloc/src/pal/pal_posix.h @@ -131,10 +131,13 @@ namespace snmalloc * high memory pressure conditions, though on Linux this seems to impose * too much of a performance penalty. */ - void notify_not_using(void* p, size_t size) noexcept + static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); #ifdef USE_POSIX_COMMIT_CHECKS + // Fill memory so that when we switch the pages back on we don't make + // assumptions on the content. + memset(p, 0x5a, size); mprotect(p, size, PROT_NONE); #else UNUSED(p); @@ -150,7 +153,7 @@ namespace snmalloc * function. */ template - void notify_using(void* p, size_t size) noexcept + static void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( is_aligned_block(p, size) || (zero_mem == NoZero)); @@ -163,7 +166,7 @@ namespace snmalloc #endif if constexpr (zero_mem == YesZero) - static_cast(this)->template zero(p, size); + zero(p, size); } /** @@ -178,7 +181,7 @@ namespace snmalloc * calling bzero at some point. */ template - void zero(void* p, size_t size) noexcept + static void zero(void* p, size_t size) noexcept { if (page_aligned || is_aligned_block(p, size)) { @@ -207,7 +210,7 @@ namespace snmalloc * POSIX does not define a portable interface for specifying alignment * greater than a page. */ - std::pair reserve_at_least(size_t size) noexcept + static std::pair reserve_at_least(size_t size) noexcept { SNMALLOC_ASSERT(size == bits::next_pow2(size)); diff --git a/3rdparty/snmalloc/src/pal/pal_windows.h b/3rdparty/snmalloc/src/pal/pal_windows.h index b4cec0989b1b..39e055603fc3 100644 --- a/3rdparty/snmalloc/src/pal/pal_windows.h +++ b/3rdparty/snmalloc/src/pal/pal_windows.h @@ -46,32 +46,6 @@ namespace snmalloc } public: - PALWindows() - { - // No error handling here - if this doesn't work, then we will just - // consume more memory. There's nothing sensible that we could do in - // error handling. We also leak both the low memory notification object - // handle and the wait object handle. We'll need them until the program - // exits, so there's little point doing anything else. - // - // We only try to register once. If this fails, give up. Even if we - // create multiple PAL objects, we don't want to get more than one - // callback. - if (!registered_for_notifications.exchange(true)) - { - lowMemoryObject = - CreateMemoryResourceNotification(LowMemoryResourceNotification); - HANDLE waitObject; - RegisterWaitForSingleObject( - &waitObject, - lowMemoryObject, - low_memory, - nullptr, - INFINITE, - WT_EXECUTEDEFAULT); - } - } - /** * Bitmap of PalFeatures flags indicating the optional features that this * PAL supports. This PAL supports low-memory notifications. @@ -90,7 +64,7 @@ namespace snmalloc * Check whether the low memory state is still in effect. This is an * expensive operation and should not be on any fast paths. */ - bool expensive_low_memory_check() + static bool expensive_low_memory_check() { BOOL result; QueryMemoryResourceNotification(lowMemoryObject, &result); @@ -105,6 +79,29 @@ namespace snmalloc static void register_for_low_memory_callback(PalNotificationObject* callback) { + // No error handling here - if this doesn't work, then we will just + // consume more memory. There's nothing sensible that we could do in + // error handling. We also leak both the low memory notification object + // handle and the wait object handle. We'll need them until the program + // exits, so there's little point doing anything else. + // + // We only try to register once. If this fails, give up. Even if we + // create multiple PAL objects, we don't want to get more than one + // callback. + if (!registered_for_notifications.exchange(true)) + { + lowMemoryObject = + CreateMemoryResourceNotification(LowMemoryResourceNotification); + HANDLE waitObject; + RegisterWaitForSingleObject( + &waitObject, + lowMemoryObject, + low_memory, + nullptr, + INFINITE, + WT_EXECUTEDEFAULT); + } + low_memory_callbacks.register_notification(callback); } @@ -116,7 +113,7 @@ namespace snmalloc } /// Notify platform that we will not be using these pages - void notify_not_using(void* p, size_t size) noexcept + static void notify_not_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT(is_aligned_block(p, size)); @@ -128,7 +125,7 @@ namespace snmalloc /// Notify platform that we will be using these pages template - void notify_using(void* p, size_t size) noexcept + static void notify_using(void* p, size_t size) noexcept { SNMALLOC_ASSERT( is_aligned_block(p, size) || (zero_mem == NoZero)); @@ -141,7 +138,7 @@ namespace snmalloc /// OS specific function for zeroing memory template - void zero(void* p, size_t size) noexcept + static void zero(void* p, size_t size) noexcept { if (page_aligned || is_aligned_block(p, size)) { @@ -154,13 +151,13 @@ namespace snmalloc } # ifdef USE_SYSTEMATIC_TESTING - size_t& systematic_bump_ptr() + static size_t& systematic_bump_ptr() { static size_t bump_ptr = (size_t)0x4000'0000'0000; return bump_ptr; } - std::pair reserve_at_least(size_t size) noexcept + static std::pair reserve_at_least(size_t size) noexcept { // Magic number for over-allocating chosen by the Pal // These should be further refined based on experiments. @@ -186,7 +183,7 @@ namespace snmalloc } # elif defined(PLATFORM_HAS_VIRTUALALLOC2) template - void* reserve_aligned(size_t size) noexcept + static void* reserve_aligned(size_t size) noexcept { SNMALLOC_ASSERT(size == bits::next_pow2(size)); SNMALLOC_ASSERT(size >= minimum_alloc_size); @@ -216,7 +213,7 @@ namespace snmalloc return ret; } # else - std::pair reserve_at_least(size_t size) noexcept + static std::pair reserve_at_least(size_t size) noexcept { SNMALLOC_ASSERT(size == bits::next_pow2(size)); diff --git a/3rdparty/snmalloc/src/test/func/bits/bits.cc b/3rdparty/snmalloc/src/test/func/bits/bits.cc new file mode 100644 index 000000000000..26c5ae7bf750 --- /dev/null +++ b/3rdparty/snmalloc/src/test/func/bits/bits.cc @@ -0,0 +1,45 @@ +/** + * Unit tests for operations in bits.h + */ + +#include +#include +#include + +void test_ctz() +{ + for (size_t i = 0; i < sizeof(size_t) * 8; i++) + if (snmalloc::bits::ctz(snmalloc::bits::one_at_bit(i)) != i) + { + std::cout << "Failed with ctz(one_at_bit(i)) != i for i=" << i + << std::endl; + abort(); + } +} + +void test_clz() +{ + const size_t PTRSIZE_LOG = sizeof(size_t) * 8; + + for (size_t i = 0; i < sizeof(size_t) * 8; i++) + if ( + snmalloc::bits::clz(snmalloc::bits::one_at_bit(i)) != + (PTRSIZE_LOG - i - 1)) + { + std::cout + << "Failed with clz(one_at_bit(i)) != (PTRSIZE_LOG - i - 1) for i=" << i + << std::endl; + abort(); + } +} + +int main(int argc, char** argv) +{ + UNUSED(argc); + UNUSED(argv); + + setup(); + + test_clz(); + test_ctz(); +} \ No newline at end of file diff --git a/3rdparty/snmalloc/src/test/func/fixed_region/fixed_region.cc b/3rdparty/snmalloc/src/test/func/fixed_region/fixed_region.cc index 8f25c575601a..5fba76a9f41a 100644 --- a/3rdparty/snmalloc/src/test/func/fixed_region/fixed_region.cc +++ b/3rdparty/snmalloc/src/test/func/fixed_region/fixed_region.cc @@ -23,6 +23,7 @@ extern "C" void oe_abort() using namespace snmalloc; int main() { +#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features auto& mp = *MemoryProviderStateMixin::make(); // 28 is large enough to produce a nested allocator. @@ -51,4 +52,5 @@ int main() if (oe_end < r1) abort(); } +#endif } diff --git a/3rdparty/snmalloc/src/test/func/malloc/malloc.cc b/3rdparty/snmalloc/src/test/func/malloc/malloc.cc index 029428d74a0a..b206cd599197 100644 --- a/3rdparty/snmalloc/src/test/func/malloc/malloc.cc +++ b/3rdparty/snmalloc/src/test/func/malloc/malloc.cc @@ -22,7 +22,15 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) const auto alloc_size = our_malloc_usable_size(p); const auto expected_size = round_size(size); - if ((align == 1) && (alloc_size != expected_size)) +#ifdef SNMALLOC_PASS_THROUGH + // Calling system allocator may allocate a larger block than + // snmalloc. Note, we have called the system allocator with + // the size snmalloc would allocate, so it won't be smaller. + const auto exact_size = false; +#else + const auto exact_size = align == 1; +#endif + if (exact_size && (alloc_size != expected_size)) { printf( "Usable size is %zu, but required to be %zu.\n", @@ -30,7 +38,7 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) expected_size); abort(); } - if ((align != 1) && (alloc_size < expected_size)) + if ((!exact_size) && (alloc_size < expected_size)) { printf( "Usable size is %zu, but required to be at least %zu.\n", @@ -46,6 +54,16 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) align); abort(); } + if ( + static_cast( + reinterpret_cast(p) % natural_alignment(size)) != 0) + { + printf( + "Address is 0x%zx, but should have natural alignment to 0x%zx.\n", + reinterpret_cast(p), + natural_alignment(size)); + abort(); + } our_free(p); } diff --git a/3rdparty/snmalloc/src/test/func/memory/memory.cc b/3rdparty/snmalloc/src/test/func/memory/memory.cc index 64cdf125b242..d12339509e7d 100644 --- a/3rdparty/snmalloc/src/test/func/memory/memory.cc +++ b/3rdparty/snmalloc/src/test/func/memory/memory.cc @@ -13,8 +13,8 @@ */ # include # include +# include # include -# include # define TEST_LIMITED # define KiB (1024ull) # define MiB (KiB * KiB) @@ -140,7 +140,7 @@ void test_random_allocation() cell = alloc->alloc(16); auto pair = allocated.insert(cell); // Check not already allocated - SNMALLOC_ASSERT(pair.second); + SNMALLOC_CHECK(pair.second); UNUSED(pair); alloc_count++; } @@ -197,14 +197,14 @@ void test_double_alloc() for (size_t i = 0; i < (n * 2); i++) { void* p = a1->alloc(20); - SNMALLOC_ASSERT(set1.find(p) == set1.end()); + SNMALLOC_CHECK(set1.find(p) == set1.end()); set1.insert(p); } for (size_t i = 0; i < (n * 2); i++) { void* p = a2->alloc(20); - SNMALLOC_ASSERT(set2.find(p) == set2.end()); + SNMALLOC_CHECK(set2.find(p) == set2.end()); set2.insert(p); } @@ -245,8 +245,8 @@ void test_external_pointer() void* p4 = Alloc::external_pointer(p2); UNUSED(p3); UNUSED(p4); - SNMALLOC_ASSERT(p1 == p3); - SNMALLOC_ASSERT((size_t)p4 == (size_t)p1 + size - 1); + SNMALLOC_CHECK(p1 == p3); + SNMALLOC_CHECK((size_t)p4 == (size_t)p1 + size - 1); } alloc->dealloc(p1, size); @@ -348,7 +348,7 @@ void test_alloc_16M() const size_t size = 16'000'000; void* p1 = alloc->alloc(size); - SNMALLOC_ASSERT(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); + SNMALLOC_CHECK(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); alloc->dealloc(p1); } @@ -359,7 +359,7 @@ void test_calloc_16M() const size_t size = 16'000'000; void* p1 = alloc->alloc(size); - SNMALLOC_ASSERT(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); + SNMALLOC_CHECK(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); alloc->dealloc(p1); } @@ -373,7 +373,7 @@ void test_calloc_large_bug() const size_t size = (SUPERSLAB_SIZE << 3) - 7; void* p1 = alloc->alloc(size); - SNMALLOC_ASSERT(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); + SNMALLOC_CHECK(Alloc::alloc_size(Alloc::external_pointer(p1)) >= size); alloc->dealloc(p1); } @@ -403,15 +403,17 @@ int main(int argc, char** argv) UNUSED(argv); #endif - test_calloc_large_bug(); - test_external_pointer_dealloc_bug(); - test_external_pointer_large(); test_alloc_dealloc_64k(); test_random_allocation(); test_calloc(); test_double_alloc(); +#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features + test_calloc_large_bug(); + test_external_pointer_dealloc_bug(); + test_external_pointer_large(); test_external_pointer(); test_alloc_16M(); test_calloc_16M(); +#endif return 0; } diff --git a/3rdparty/snmalloc/src/test/func/memory_usage/memory_usage.cc b/3rdparty/snmalloc/src/test/func/memory_usage/memory_usage.cc new file mode 100644 index 000000000000..47e4ce9c8dc4 --- /dev/null +++ b/3rdparty/snmalloc/src/test/func/memory_usage/memory_usage.cc @@ -0,0 +1,104 @@ +/** + * Memory usage test + * Query memory usage repeatedly + */ + +#include +#include +#include + +#define SNMALLOC_NAME_MANGLE(a) our_##a +#include "../../../override/malloc-extensions.cc" +#include "../../../override/malloc.cc" + +using namespace snmalloc; + +bool print_memory_usage() +{ + static malloc_info_v1 last_memory_usage; + malloc_info_v1 next_memory_usage; + + get_malloc_info_v1(&next_memory_usage); + + if ( + (next_memory_usage.current_memory_usage != + last_memory_usage.current_memory_usage) || + (next_memory_usage.peak_memory_usage != + last_memory_usage.peak_memory_usage)) + { + std::cout << "Memory Usages Changed to (" + << next_memory_usage.current_memory_usage << ", " + << next_memory_usage.peak_memory_usage << ")" << std::endl; + last_memory_usage = next_memory_usage; + return true; + } + return false; +} + +std::vector allocs; + +/** + * Add allocs until the statistics have changed n times. + */ +void add_n_allocs(size_t n) +{ + while (true) + { + allocs.push_back(our_malloc(1024)); + if (print_memory_usage()) + { + n--; + if (n == 0) + break; + } + } +} + +/** + * Remove allocs until the statistics have changed n times. + */ +void remove_n_allocs(size_t n) +{ + while (true) + { + our_free(allocs.back()); + allocs.pop_back(); + if (print_memory_usage()) + { + n--; + if (n == 0) + break; + } + } +} + +int main(int argc, char** argv) +{ + UNUSED(argc); + UNUSED(argv); +#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features + setup(); + + add_n_allocs(5); + std::cout << "Init complete!" << std::endl; + + for (int i = 0; i < 10; i++) + { + remove_n_allocs(1); + std::cout << "Phase " << i << " remove complete!" << std::endl; + add_n_allocs(2); + std::cout << "Phase " << i << " add complete!" << std::endl; + } + + for (int i = 0; i < 10; i++) + { + remove_n_allocs(2); + std::cout << "Phase " << i << " remove complete!" << std::endl; + add_n_allocs(1); + std::cout << "Phase " << i << " add complete!" << std::endl; + } + + remove_n_allocs(3); + std::cout << "Teardown complete!" << std::endl; +#endif +} diff --git a/3rdparty/snmalloc/src/test/func/pagemap/pagemap.cc b/3rdparty/snmalloc/src/test/func/pagemap/pagemap.cc new file mode 100644 index 000000000000..571cff09ba0f --- /dev/null +++ b/3rdparty/snmalloc/src/test/func/pagemap/pagemap.cc @@ -0,0 +1,55 @@ +/** + * Unit tests for operations in pagemap operations. + * + * Currently this tests a very specific case where the pagemap + * requires multiple levels of index. This was incorrectly implemented, + * but no examples were using multiple levels of pagemap. + */ + +#include +#include +#include +#include + +using namespace snmalloc; +using T = size_t; +static constexpr size_t GRANULARITY_BITS = 9; +static constexpr T PRIME = 251; +Pagemap pagemap_test; + +int main(int argc, char** argv) +{ + UNUSED(argc); + UNUSED(argv); + + setup(); + + T value = 0; + for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36); + ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) + { + pagemap_test.set(ptr, value); + value++; + if (value == PRIME) + value = 0; + if ((ptr % (1ULL << 32)) == 0) + std::cout << "." << std::flush; + } + + std::cout << std::endl; + value = 0; + for (uintptr_t ptr = 0; ptr < bits::one_at_bit(36); + ptr += bits::one_at_bit(GRANULARITY_BITS + 3)) + { + T result = pagemap_test.get(ptr); + if (value != result) + Pal::error("Pagemap corrupt!"); + value++; + if (value == PRIME) + value = 0; + + if ((ptr % (1ULL << 32)) == 0) + std::cout << "." << std::flush; + } + std::cout << std::endl; +} diff --git a/3rdparty/snmalloc/src/test/func/sizeclass/sizeclass.cc b/3rdparty/snmalloc/src/test/func/sizeclass/sizeclass.cc index 7d6eaf817a20..696d0140b23f 100644 --- a/3rdparty/snmalloc/src/test/func/sizeclass/sizeclass.cc +++ b/3rdparty/snmalloc/src/test/func/sizeclass/sizeclass.cc @@ -12,7 +12,7 @@ void test_align_size() { bool failed = false; - SNMALLOC_ASSERT(snmalloc::aligned_size(128, 160) == 256); + SNMALLOC_CHECK(snmalloc::aligned_size(128, 160) == 256); for (size_t size = 1; size < snmalloc::sizeclass_to_size(snmalloc::NUM_SIZECLASSES - 1); diff --git a/3rdparty/snmalloc/src/test/func/statistics/stats.cc b/3rdparty/snmalloc/src/test/func/statistics/stats.cc index 6ecbe09ab45b..bd9dfec86cfd 100644 --- a/3rdparty/snmalloc/src/test/func/statistics/stats.cc +++ b/3rdparty/snmalloc/src/test/func/statistics/stats.cc @@ -2,6 +2,7 @@ int main() { +#ifndef SNMALLOC_PASS_THROUGH // This test depends on snmalloc internals snmalloc::Alloc* a = snmalloc::ThreadAlloc::get(); bool result; @@ -36,4 +37,5 @@ int main() { abort(); } +#endif } \ No newline at end of file diff --git a/3rdparty/snmalloc/src/test/perf/external_pointer/externalpointer.cc b/3rdparty/snmalloc/src/test/perf/external_pointer/externalpointer.cc index 20e7bbaa82fb..513d3de62dbd 100644 --- a/3rdparty/snmalloc/src/test/perf/external_pointer/externalpointer.cc +++ b/3rdparty/snmalloc/src/test/perf/external_pointer/externalpointer.cc @@ -85,16 +85,18 @@ namespace test int main(int, char**) { +#ifndef SNMALLOC_PASS_THROUGH // Depends on snmalloc specific features setup(); xoroshiro::p128r64 r; -#ifdef NDEBUG +# ifdef NDEBUG size_t nn = 30; -#else +# else size_t nn = 3; -#endif +# endif for (size_t n = 0; n < nn; n++) test::test_external_pointer(r); return 0; +#endif } diff --git a/3rdparty/snmalloc/src/test/perf/low_memory/low-memory.cc b/3rdparty/snmalloc/src/test/perf/low_memory/low-memory.cc index 68007b4a72fb..83d8742d12f9 100644 --- a/3rdparty/snmalloc/src/test/perf/low_memory/low-memory.cc +++ b/3rdparty/snmalloc/src/test/perf/low_memory/low-memory.cc @@ -98,17 +98,17 @@ void reduce_pressure(Queue& allocations) * Template parameter required to handle `if constexpr` always evaluating both * sides. */ -template +template void register_for_pal_notifications() { - PAL::register_for_low_memory_callback(&update_epoch); + MemoryProvider::Pal::register_for_low_memory_callback(&update_epoch); } int main(int argc, char** argv) { opt::Opt opt(argc, argv); - if constexpr (pal_supports) + if constexpr (pal_supports) { register_for_pal_notifications(); } diff --git a/3rdparty/snmalloc/src/test/perf/singlethread/singlethread.cc b/3rdparty/snmalloc/src/test/perf/singlethread/singlethread.cc index 97053ce65041..00880282d146 100644 --- a/3rdparty/snmalloc/src/test/perf/singlethread/singlethread.cc +++ b/3rdparty/snmalloc/src/test/perf/singlethread/singlethread.cc @@ -20,7 +20,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) for (size_t i = 0; i < ((count * 3) / 2); i++) { void* p = alloc->alloc(size); - SNMALLOC_ASSERT(set.find(p) == set.end()); + SNMALLOC_CHECK(set.find(p) == set.end()); if (write) *(int*)p = 4; @@ -35,14 +35,14 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) void* p = *it; alloc->dealloc(p, size); set.erase(it); - SNMALLOC_ASSERT(set.find(p) == set.end()); + SNMALLOC_CHECK(set.find(p) == set.end()); } // alloc 1x objects for (size_t i = 0; i < count; i++) { void* p = alloc->alloc(size); - SNMALLOC_ASSERT(set.find(p) == set.end()); + SNMALLOC_CHECK(set.find(p) == set.end()); if (write) *(int*)p = 4; diff --git a/cgmanifest.json b/cgmanifest.json index bcd163f33bfc..78437db1ab54 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -87,7 +87,7 @@ "type": "git", "git": { "repositoryUrl": "https://github.com/microsoft/snmalloc", - "commitHash": "4e1f5829a754ab9c170ec090979d3a670d2d5d1a" + "commitHash": "3ca29d6f501ba5d2608b99e6afa471e1e91001e8" } } },