diff --git a/.github/workflows/monitor-components.yml b/.github/workflows/monitor-components.yml index a074403c64f42d..f15ff218d28b81 100644 --- a/.github/workflows/monitor-components.yml +++ b/.github/workflows/monitor-components.yml @@ -79,7 +79,7 @@ jobs: feed: https://github.com/jrsoftware/issrc/tags.atom - label: mimalloc feed: https://github.com/microsoft/mimalloc/tags.atom - title-pattern: ^(?!v1\.) + title-pattern: ^(?!v1\.|v3\.[01]\.) fail-fast: false steps: - uses: git-for-windows/rss-to-issues@v0 diff --git a/compat/mimalloc/alloc-aligned.c b/compat/mimalloc/alloc-aligned.c index 3d3202eb574971..772b76c2027944 100644 --- a/compat/mimalloc/alloc-aligned.c +++ b/compat/mimalloc/alloc-aligned.c @@ -37,22 +37,22 @@ static mi_decl_restrict void* mi_heap_malloc_guarded_aligned(mi_heap_t* heap, si return p; } -static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero, size_t* usable) { const size_t rate = heap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_heap_empty` if (rate != 0) { heap->guarded_sample_rate = 0; } - void* p = _mi_heap_malloc_zero(heap, size, zero); + void* p = _mi_heap_malloc_zero_ex(heap, size, zero, 0, usable); if (rate != 0) { heap->guarded_sample_rate = rate; } return p; } #else -static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero) { - return _mi_heap_malloc_zero(heap, size, zero); +static void* mi_heap_malloc_zero_no_guarded(mi_heap_t* heap, size_t size, bool zero, size_t* usable) { + return _mi_heap_malloc_zero_ex(heap, size, zero, 0, usable); } #endif // Fallback aligned allocation that over-allocates -- split out for better codegen -static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept { mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)); mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); @@ -72,14 +72,14 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment); // the page block size should be large enough to align in the single huge page block + p = _mi_heap_malloc_zero_ex(heap, oversize, false, alignment, usable); // the page block size should be large enough to align in the single huge page block // zero afterwards as only the area from the aligned_p may be committed! if (p == NULL) return NULL; } else { // otherwise over-allocate oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and aligment 64k, we would allocate a 64k block and pointing just beyond that. - p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero); + p = mi_heap_malloc_zero_no_guarded(heap, oversize, zero, usable); if (p == NULL) return NULL; } mi_page_t* page = _mi_ptr_page(p); @@ -132,7 +132,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_overalloc(mi_heap_t } // Generic primitive aligned allocation -- split out for better codegen -static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept { mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) @@ -147,7 +147,7 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* // this is important to try as the fast path in `mi_heap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero); + void* p = mi_heap_malloc_zero_no_guarded(heap, size, zero, usable); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { @@ -161,12 +161,14 @@ static mi_decl_noinline void* mi_heap_malloc_zero_aligned_at_generic(mi_heap_t* } // fall back to over-allocation - return mi_heap_malloc_zero_aligned_at_overalloc(heap,size,alignment,offset,zero); + return mi_heap_malloc_zero_aligned_at_overalloc(heap,size,alignment,offset,zero,usable); } // Primitive aligned allocation -static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, const size_t alignment, const size_t offset, const bool zero) mi_attr_noexcept +static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t size, + const size_t alignment, const size_t offset, const bool zero, + size_t* usable) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see ) @@ -191,6 +193,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) { + if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } void* p = (zero ? _mi_page_malloc_zeroed(heap,page,padsize) : _mi_page_malloc(heap,page,padsize)); // call specific page malloc for better codegen mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); @@ -201,7 +204,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t } // fallback to generic aligned allocation - return mi_heap_malloc_zero_aligned_at_generic(heap, size, alignment, offset, zero); + return mi_heap_malloc_zero_aligned_at_generic(heap, size, alignment, offset, zero, usable); } @@ -210,7 +213,7 @@ static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* const heap, const size_t // ------------------------------------------------------ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); + return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { @@ -227,7 +230,7 @@ void* _mi_extern_heap_malloc_aligned = (void*)&mi_heap_malloc_aligned; // ------------------------------------------------------ mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { - return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); + return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { @@ -252,6 +255,10 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a return mi_heap_malloc_aligned(mi_prim_get_default_heap(), size, alignment); } +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept { + return mi_heap_malloc_zero_aligned_at(mi_prim_get_default_heap(), size, alignment, 0, false, block_size); +} + mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_zalloc_aligned_at(mi_prim_get_default_heap(), size, alignment, offset); } @@ -260,6 +267,10 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a return mi_heap_zalloc_aligned(mi_prim_get_default_heap(), size, alignment); } +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept { + return mi_heap_malloc_zero_aligned_at(mi_prim_get_default_heap(), size, alignment, 0, true, block_size); +} + mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { return mi_heap_calloc_aligned_at(mi_prim_get_default_heap(), count, size, alignment, offset); } @@ -275,8 +286,8 @@ mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); - if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero); + if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero,NULL,NULL); + if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero,NULL); size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) % alignment) == 0) { @@ -300,7 +311,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); + if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero,NULL,NULL); size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL) return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero); } diff --git a/compat/mimalloc/alloc.c b/compat/mimalloc/alloc.c index f975a92b6b6d1a..b6cdca1882ecb7 100644 --- a/compat/mimalloc/alloc.c +++ b/compat/mimalloc/alloc.c @@ -27,7 +27,7 @@ terms of the MIT license. A copy of the license can be found in the file // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept +extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept { mi_assert_internal(size >= MI_PADDING_SIZE); mi_assert_internal(page->block_size == 0 /* empty heap */ || mi_page_block_size(page) >= size); @@ -35,10 +35,10 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ // check the free list mi_block_t* const block = page->free; if mi_unlikely(block == NULL) { - return _mi_malloc_generic(heap, size, zero, 0); + return _mi_malloc_generic(heap, size, zero, 0, usable); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); - + if (usable != NULL) { *usable = mi_page_usable_block_size(page); }; // pop from the free list page->free = mi_block_next(page, block); page->used++; @@ -116,17 +116,17 @@ extern inline void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_ // extra entries for improved efficiency in `alloc-aligned.c`. extern void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept { - return _mi_page_malloc_zero(heap,page,size,false); + return _mi_page_malloc_zero(heap,page,size,false,NULL); } extern void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept { - return _mi_page_malloc_zero(heap,page,size,true); + return _mi_page_malloc_zero(heap,page,size,true,NULL); } #if MI_GUARDED mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; #endif -static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { +static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, size_t size, bool zero, size_t* usable) mi_attr_noexcept { mi_assert(heap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG @@ -144,7 +144,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, // get page in constant time, and allocate from it mi_page_t* page = _mi_heap_get_free_small_page(heap, size + MI_PADDING_SIZE); - void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero); + void* const p = _mi_page_malloc_zero(heap, page, size + MI_PADDING_SIZE, zero, usable); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -157,7 +157,7 @@ static inline mi_decl_restrict void* mi_heap_malloc_small_zero(mi_heap_t* heap, // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_heap_malloc_small_zero(heap, size, false); + return mi_heap_malloc_small_zero(heap, size, false, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { @@ -165,11 +165,11 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc_small(size_t si } // The main allocation function -extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept { +extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept { // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_heap_malloc_small_zero(heap, size, zero); + return mi_heap_malloc_small_zero(heap, size, zero, usable); } #if MI_GUARDED else if (huge_alignment==0 && mi_heap_malloc_use_guarded(heap,size)) { @@ -180,7 +180,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z // regular allocation mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local - void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(heap, size + MI_PADDING_SIZE, zero, huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -193,7 +193,7 @@ extern inline void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool z } extern inline void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept { - return _mi_heap_malloc_zero_ex(heap, size, zero, 0); + return _mi_heap_malloc_zero_ex(heap, size, zero, 0, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -206,7 +206,7 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_malloc(size_t size) mi // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, true); + return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -228,6 +228,29 @@ mi_decl_nodiscard mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi return mi_heap_calloc(mi_prim_get_default_heap(),count,size); } +// Return usable size +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept { + return mi_heap_malloc_small_zero(mi_prim_get_default_heap(), size, false, usable); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_heap_umalloc(mi_heap_t* heap, size_t size, size_t* usable) mi_attr_noexcept { + return _mi_heap_malloc_zero_ex(heap, size, false, 0, usable); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept { + return mi_heap_umalloc(mi_prim_get_default_heap(), size, usable); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept { + return _mi_heap_malloc_zero_ex(mi_prim_get_default_heap(), size, true, 0, usable); +} + +mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept { + size_t total; + if (mi_count_size_overflow(count,size,&total)) return NULL; + return mi_uzalloc(total, usable); +} + // Uninitialized `calloc` mi_decl_nodiscard extern mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { size_t total; @@ -247,25 +270,38 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { return NULL; #else if (p == NULL) return NULL; - const size_t size = _mi_usable_size(p,"mi_expand"); + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); + const size_t size = _mi_usable_size(p,page); if (newsize > size) return NULL; return p; // it fits #endif } -void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept { +void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept { // if p == NULL then behave as malloc. // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) - const size_t size = _mi_usable_size(p,"mi_realloc"); // also works if p == NULL (with size 0) + const mi_page_t* page; + size_t size; + if (p==NULL) { + page = NULL; + size = 0; + if (usable_pre!=NULL) { *usable_pre = 0; } + } + else { + page = mi_validate_ptr_page(p,"mi_realloc"); + size = _mi_usable_size(p,page); + if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); } + } if mi_unlikely(newsize <= size && newsize >= (size / 2) && newsize > 0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } + if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); } return p; // reallocation still fits and not more than 50% waste } - void* newp = mi_heap_malloc(heap,newsize); + void* newp = mi_heap_umalloc(heap,newsize,usable_post); if mi_likely(newp != NULL) { if (zero && newsize > size) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized @@ -286,7 +322,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) } mi_decl_nodiscard void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { - return _mi_heap_realloc_zero(heap, p, newsize, false); + return _mi_heap_realloc_zero(heap, p, newsize, false, NULL, NULL); } mi_decl_nodiscard void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { @@ -304,7 +340,7 @@ mi_decl_nodiscard void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsiz } mi_decl_nodiscard void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { - return _mi_heap_realloc_zero(heap, p, newsize, true); + return _mi_heap_realloc_zero(heap, p, newsize, true, NULL, NULL); } mi_decl_nodiscard void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { @@ -322,6 +358,10 @@ mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_ return mi_heap_reallocn(mi_prim_get_default_heap(),p,count,size); } +mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept { + return _mi_heap_realloc_zero(mi_prim_get_default_heap(),p,newsize, false, usable_pre, usable_post); +} + // Reallocate but free `p` on errors mi_decl_nodiscard void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_reallocf(mi_prim_get_default_heap(),p,newsize); @@ -615,8 +655,11 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size) { } uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); - if (segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) { - _mi_os_protect(guard_page, os_page_size); + if mi_likely(segment->allow_decommit && _mi_is_aligned(guard_page, os_page_size)) { + const bool ok = _mi_os_protect(guard_page, os_page_size); + if mi_unlikely(!ok) { + _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); + } } else { _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); @@ -646,7 +689,7 @@ mi_decl_restrict void* _mi_heap_malloc_guarded(mi_heap_t* heap, size_t size, boo const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(heap, req_size, zero, 0 /* huge_alignment */, NULL); if (block==NULL) return NULL; void* const p = mi_block_ptr_set_guarded(block, obj_size); diff --git a/compat/mimalloc/arena.c b/compat/mimalloc/arena.c index e97ca885fed86a..c87dd23b54107c 100644 --- a/compat/mimalloc/arena.c +++ b/compat/mimalloc/arena.c @@ -798,16 +798,18 @@ static bool mi_arena_add(mi_arena_t* arena, mi_arena_id_t* arena_id, mi_stats_t* mi_assert_internal(arena->block_count > 0); if (arena_id != NULL) { *arena_id = -1; } - size_t i = mi_atomic_increment_acq_rel(&mi_arena_count); - if (i >= MI_MAX_ARENAS) { - mi_atomic_decrement_acq_rel(&mi_arena_count); - return false; + size_t i = mi_atomic_load_relaxed(&mi_arena_count); + while (i < MI_MAX_ARENAS) { + if (mi_atomic_cas_strong_acq_rel(&mi_arena_count, &i, i+1)) { + _mi_stat_counter_increase(&stats->arena_count, 1); + arena->id = mi_arena_id_create(i); + mi_atomic_store_ptr_release(mi_arena_t, &mi_arenas[i], arena); + if (arena_id != NULL) { *arena_id = arena->id; } + return true; + } } - _mi_stat_counter_increase(&stats->arena_count,1); - arena->id = mi_arena_id_create(i); - mi_atomic_store_ptr_release(mi_arena_t,&mi_arenas[i], arena); - if (arena_id != NULL) { *arena_id = arena->id; } - return true; + + return false; } static bool mi_manage_os_memory_ex2(void* start, size_t size, bool is_large, int numa_node, bool exclusive, mi_memid_t memid, mi_arena_id_t* arena_id) mi_attr_noexcept diff --git a/compat/mimalloc/free.c b/compat/mimalloc/free.c index 5e5ae443f3a3a3..0129ce83bd6c06 100644 --- a/compat/mimalloc/free.c +++ b/compat/mimalloc/free.c @@ -148,14 +148,15 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms // Free a block // Fast path written carefully to prevent register spilling on the stack -void mi_free(void* p) mi_attr_noexcept +static inline void mi_free_ex(void* p, size_t* usable) mi_attr_noexcept { mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); if mi_unlikely(segment==NULL) return; const bool is_local = (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); mi_page_t* const page = _mi_segment_page_of(segment, p); - + if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if mi_likely(is_local) { // thread-local free? if mi_likely(page->flags.full_aligned == 0) { // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned) // thread-local, aligned, and not a full page @@ -173,6 +174,14 @@ void mi_free(void* p) mi_attr_noexcept } } +void mi_free(void* p) mi_attr_noexcept { + mi_free_ex(p,NULL); +} + +void mi_ufree(void* p, size_t* usable) mi_attr_noexcept { + mi_free_ex(p,usable); +} + // return true if successful bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page @@ -323,10 +332,15 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p return aligned_size; } -static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { +static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) { const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); - if mi_unlikely(segment==NULL) return 0; - const mi_page_t* const page = _mi_segment_page_of(segment, p); + if mi_unlikely(segment==NULL) return NULL; + mi_page_t* const page = _mi_segment_page_of(segment, p); + return page; +} + +static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept { + if mi_unlikely(page==NULL) return 0; if mi_likely(!mi_page_has_aligned(page)) { const mi_block_t* block = (const mi_block_t*)p; return mi_page_usable_size_of(page, block); @@ -338,7 +352,8 @@ static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noe } mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { - return _mi_usable_size(p, "mi_usable_size"); + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size"); + return _mi_usable_size(p,page); } @@ -349,7 +364,8 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { void mi_free_size(void* p, size_t size) mi_attr_noexcept { MI_UNUSED_RELEASE(size); #if MI_DEBUG - const size_t available = _mi_usable_size(p,"mi_free_size"); + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); + const size_t available = _mi_usable_size(p,page); mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ ); #endif mi_free(p); diff --git a/compat/mimalloc/heap.c b/compat/mimalloc/heap.c index f96e60d0f8d94c..88969311e89586 100644 --- a/compat/mimalloc/heap.c +++ b/compat/mimalloc/heap.c @@ -223,7 +223,11 @@ void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool heap->no_reclaim = noreclaim; heap->tag = tag; if (heap == tld->heap_backing) { - _mi_random_init(&heap->random); + #if defined(_WIN32) && !defined(MI_SHARED_LIB) + _mi_random_init_weak(&heap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) + #else + _mi_random_init(&heap->random); + #endif } else { _mi_random_split(&tld->heap_backing->random, &heap->random); diff --git a/compat/mimalloc/init.c b/compat/mimalloc/init.c index 3fc8b033695a38..c6cca89da9c5db 100644 --- a/compat/mimalloc/init.c +++ b/compat/mimalloc/init.c @@ -71,13 +71,14 @@ const mi_page_t _mi_page_empty = { // Empty statistics #define MI_STATS_NULL \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ - MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ + { 0 }, { 0 }, \ + MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ { 0 }, { 0 }, { 0 }, { 0 }, \ { 0 }, { 0 }, { 0 }, { 0 }, \ \ - { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, \ + { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, \ MI_INIT4(MI_STAT_COUNT_NULL), \ { 0 }, { 0 }, { 0 }, { 0 }, \ \ @@ -649,7 +650,7 @@ void mi_process_init(void) mi_attr_noexcept { if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); - long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at); + int reserve_at = (int)mi_option_get_clamp(mi_option_reserve_huge_os_pages_at, -1, INT_MAX); if (reserve_at != -1) { mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); } else { diff --git a/compat/mimalloc/mimalloc-stats.h b/compat/mimalloc/mimalloc-stats.h index 44c4886f88a0c7..12c5c9a7d6ced7 100644 --- a/compat/mimalloc/mimalloc-stats.h +++ b/compat/mimalloc/mimalloc-stats.h @@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #include -#define MI_STAT_VERSION 1 // increased on every backward incompatible change +#define MI_STAT_VERSION 3 // increased on every backward incompatible change // count allocation over time typedef struct mi_stat_count_s { @@ -29,8 +29,8 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNT(pages) /* count of mimalloc pages */ \ MI_STAT_COUNT(reserved) /* reserved memory bytes */ \ MI_STAT_COUNT(committed) /* committed bytes */ \ - MI_STAT_COUNT(reset) /* reset bytes */ \ - MI_STAT_COUNT(purged) /* purged bytes */ \ + MI_STAT_COUNTER(reset) /* reset bytes */ \ + MI_STAT_COUNTER(purged) /* purged bytes */ \ MI_STAT_COUNT(page_committed) /* committed memory inside pages */ \ MI_STAT_COUNT(pages_abandoned) /* abandonded pages count */ \ MI_STAT_COUNT(threads) /* number of threads */ \ @@ -52,7 +52,8 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNTER(arena_purges) \ MI_STAT_COUNTER(pages_extended) /* number of page extensions */ \ MI_STAT_COUNTER(pages_retire) /* number of pages that are retired */ \ - MI_STAT_COUNTER(page_searches) /* searches for a fresh page */ \ + MI_STAT_COUNTER(page_searches) /* total pages searched for a fresh page */ \ + MI_STAT_COUNTER(page_searches_count) /* searched count for a fresh page */ \ /* only on v1 and v2 */ \ MI_STAT_COUNT(segments) \ MI_STAT_COUNT(segments_abandoned) \ diff --git a/compat/mimalloc/mimalloc.h b/compat/mimalloc/mimalloc.h index 19829516d42c1f..fcd19cc9bc2e15 100644 --- a/compat/mimalloc/mimalloc.h +++ b/compat/mimalloc/mimalloc.h @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 224 // major + 2 digits minor +#define MI_MALLOC_VERSION 226 // major + 2 digits minor // ------------------------------------------------------ // Compiler specific attributes @@ -186,6 +186,22 @@ mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsiz mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); +// ----------------------------------------------------------------- +// Return allocated block size (if the return value is not NULL) +// ----------------------------------------------------------------- + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); +mi_decl_nodiscard mi_decl_export void* mi_urealloc(void* p, size_t newsize, size_t* block_size_pre, size_t* block_size_post) mi_attr_noexcept mi_attr_alloc_size(2); +mi_decl_export void mi_ufree(void* p, size_t* block_size) mi_attr_noexcept; + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); + +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* block_size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); + + // ------------------------------------------------------------------------------------- // Heaps: first-class, but can only allocate from the same thread that created it. // ------------------------------------------------------------------------------------- @@ -361,7 +377,7 @@ typedef enum mi_option_e { mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1) mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit) - mi_option_allow_large_os_pages, // allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process. + mi_option_allow_large_os_pages, // allow use of large (2 or 4 MiB) OS pages, implies eager commit. mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`) @@ -392,6 +408,7 @@ typedef enum mi_option_e { mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_target_segments_per_thread, // experimental (=0) mi_option_generic_collect, // collect heaps every N (=10000) generic allocation calls + mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/compat/mimalloc/mimalloc/internal.h b/compat/mimalloc/mimalloc/internal.h index ca5be9304a8bae..6845c9b5df3faf 100644 --- a/compat/mimalloc/mimalloc/internal.h +++ b/compat/mimalloc/mimalloc/internal.h @@ -177,7 +177,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, boo void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid); void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); -bool _mi_os_use_large_page(size_t size, size_t alignment); +bool _mi_os_canuse_large_page(size_t size, size_t alignment); size_t _mi_os_large_page_size(void); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); @@ -238,7 +238,7 @@ bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment); bool _mi_segment_visit_blocks(mi_segment_t* segment, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); // "page.c" -void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -258,9 +258,9 @@ void _mi_deferred_free(mi_heap_t* heap, bool force); void _mi_page_free_collect(mi_page_t* page,bool force); void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments -size_t _mi_page_bin(const mi_page_t* page); // for stats -size_t _mi_bin_size(size_t bin); // for stats -size_t _mi_bin(size_t size); // for stats +size_t _mi_page_stats_bin(const mi_page_t* page); // for stats +size_t _mi_bin_size(size_t bin); // for stats +size_t _mi_bin(size_t size); // for stats // "heap.c" void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag); @@ -281,12 +281,12 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); // "alloc.c" -void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_malloc_generic` +void* _mi_page_malloc_zero(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero, size_t* usable) mi_attr_noexcept; // called from `_mi_malloc_generic` void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` void* _mi_page_malloc_zeroed(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; -void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` -void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept; +void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` +void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); bool _mi_free_delayed_block(mi_block_t* block); void _mi_free_generic(mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration diff --git a/compat/mimalloc/mimalloc/prim.h b/compat/mimalloc/mimalloc/prim.h index 1087d9b8dad304..f8abc8c48cea32 100644 --- a/compat/mimalloc/mimalloc/prim.h +++ b/compat/mimalloc/mimalloc/prim.h @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file #pragma once #ifndef MIMALLOC_PRIM_H #define MIMALLOC_PRIM_H - +#include "internal.h" // mi_decl_hidden // -------------------------------------------------------------------------- // This file specifies the primitive portability API. diff --git a/compat/mimalloc/mimalloc/types.h b/compat/mimalloc/mimalloc/types.h index a15d9cba4658cb..f52d37a82b19b6 100644 --- a/compat/mimalloc/mimalloc/types.h +++ b/compat/mimalloc/mimalloc/types.h @@ -25,6 +25,7 @@ terms of the MIT license. A copy of the license can be found in the file #include #include // ptrdiff_t #include // uintptr_t, uint16_t, etc +#include // bool #include "atomic.h" // _Atomic #ifdef _MSC_VER diff --git a/compat/mimalloc/options.c b/compat/mimalloc/options.c index af2a0e70c4c7ac..b07f029e65dd29 100644 --- a/compat/mimalloc/options.c +++ b/compat/mimalloc/options.c @@ -79,12 +79,8 @@ typedef struct mi_option_desc_s { #endif #ifndef MI_DEFAULT_ALLOW_LARGE_OS_PAGES -#if defined(__linux__) && !defined(__ANDROID__) -#define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 2 // enabled, but only use transparent huge pages through madvise -#else #define MI_DEFAULT_ALLOW_LARGE_OS_PAGES 0 #endif -#endif #ifndef MI_DEFAULT_RESERVE_HUGE_OS_PAGES #define MI_DEFAULT_RESERVE_HUGE_OS_PAGES 0 @@ -103,6 +99,15 @@ typedef struct mi_option_desc_s { #endif +#ifndef MI_DEFAULT_ALLOW_THP +#if defined(__ANDROID__) +#define MI_DEFAULT_ALLOW_THP 0 +#else +#define MI_DEFAULT_ALLOW_THP 1 +#endif +#endif + +// Static options static mi_option_desc_t options[_mi_option_last] = { // stable options @@ -163,6 +168,8 @@ static mi_option_desc_t options[_mi_option_last] = { 0, UNINIT, MI_OPTION(guarded_sample_seed)}, { 0, UNINIT, MI_OPTION(target_segments_per_thread) }, // abandon segments beyond this point, or 0 to disable. { 10000, UNINIT, MI_OPTION(generic_collect) }, // collect heaps every N (=10000) generic allocation calls + { MI_DEFAULT_ALLOW_THP, + UNINIT, MI_OPTION(allow_thp) } // allow transparent huge pages? }; static void mi_option_init(mi_option_desc_t* desc); diff --git a/compat/mimalloc/os.c b/compat/mimalloc/os.c index 9b1b4b460775f1..241d6a2bee3487 100644 --- a/compat/mimalloc/os.c +++ b/compat/mimalloc/os.c @@ -62,9 +62,9 @@ size_t _mi_os_large_page_size(void) { return (mi_os_mem_config.large_page_size != 0 ? mi_os_mem_config.large_page_size : _mi_os_page_size()); } -bool _mi_os_use_large_page(size_t size, size_t alignment) { +bool _mi_os_canuse_large_page(size_t size, size_t alignment) { // if we have access, check the size and alignment requirements - if (mi_os_mem_config.large_page_size == 0 || !mi_option_is_enabled(mi_option_allow_large_os_pages)) return false; + if (mi_os_mem_config.large_page_size == 0) return false; return ((size % mi_os_mem_config.large_page_size) == 0 && (alignment % mi_os_mem_config.large_page_size) == 0); } @@ -329,7 +329,7 @@ void* _mi_os_alloc(size_t size, mi_memid_t* memid) { void* p = mi_os_prim_alloc(size, 0, true, false, &os_is_large, &os_is_zero); if (p == NULL) return NULL; - *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); + *memid = _mi_memid_create_os(p, size, true, os_is_zero, os_is_large); mi_assert_internal(memid->mem.os.size >= size); mi_assert_internal(memid->initially_committed); return p; @@ -355,7 +355,7 @@ void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allo mi_assert_internal(memid->mem.os.size >= size); mi_assert_internal(_mi_is_aligned(p,alignment)); - if (commit) { mi_assert_internal(memid->initially_committed); } + if (commit) { mi_assert_internal(memid->initially_committed); } return p; } @@ -448,7 +448,6 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) { if (is_zero != NULL) { *is_zero = false; } - mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit mi_os_stat_counter_increase(commit_calls, 1); // page align range @@ -472,6 +471,7 @@ bool _mi_os_commit_ex(void* addr, size_t size, bool* is_zero, size_t stat_size) if (os_is_zero) { mi_track_mem_defined(start,csize); } else { mi_track_mem_undefined(start,csize); } #endif + mi_os_stat_increase(committed, stat_size); // use size for precise commit vs. decommit return true; } @@ -513,7 +513,7 @@ bool _mi_os_reset(void* addr, size_t size) { size_t csize; void* start = mi_os_page_align_area_conservative(addr, size, &csize); if (csize == 0) return true; // || _mi_os_is_huge_reserved(addr) - mi_os_stat_increase(reset, csize); + mi_os_stat_counter_increase(reset, csize); mi_os_stat_counter_increase(reset_calls, 1); #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN @@ -545,7 +545,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size) { if (mi_option_get(mi_option_purge_delay) < 0) return false; // is purging allowed? mi_os_stat_counter_increase(purge_calls, 1); - mi_os_stat_increase(purged, size); + mi_os_stat_counter_increase(purged, size); if (mi_option_is_enabled(mi_option_purge_decommits) && // should decommit? !_mi_preloading()) // don't decommit during preloading (unsafe) diff --git a/compat/mimalloc/page-queue.c b/compat/mimalloc/page-queue.c index c719b6265afe54..1f700c6df4c866 100644 --- a/compat/mimalloc/page-queue.c +++ b/compat/mimalloc/page-queue.c @@ -140,15 +140,22 @@ static inline bool mi_page_is_large_or_huge(const mi_page_t* page) { return (mi_page_block_size(page) > MI_MEDIUM_OBJ_SIZE_MAX || mi_page_is_huge(page)); } -size_t _mi_page_bin(const mi_page_t* page) { +static size_t mi_page_bin(const mi_page_t* page) { const size_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page)))); mi_assert_internal(bin <= MI_BIN_FULL); return bin; } +// returns the page bin without using MI_BIN_FULL for statistics +size_t _mi_page_stats_bin(const mi_page_t* page) { + const size_t bin = (mi_page_is_huge(page) ? MI_BIN_HUGE : mi_bin(mi_page_block_size(page))); + mi_assert_internal(bin <= MI_BIN_HUGE); + return bin; +} + static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { mi_assert_internal(heap!=NULL); - const size_t bin = _mi_page_bin(page); + const size_t bin = mi_page_bin(page); mi_page_queue_t* pq = &heap->pages[bin]; mi_assert_internal((mi_page_block_size(page) == pq->block_size) || (mi_page_is_large_or_huge(page) && mi_page_queue_is_huge(pq)) || diff --git a/compat/mimalloc/page.c b/compat/mimalloc/page.c index a5a10503248622..34dae9f5e473cb 100644 --- a/compat/mimalloc/page.c +++ b/compat/mimalloc/page.c @@ -291,7 +291,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size mi_assert_internal(full_block_size >= block_size); mi_page_init(heap, page, full_block_size, heap->tld); mi_heap_stat_increase(heap, pages, 1); - mi_heap_stat_increase(heap, page_bins[_mi_page_bin(page)], 1); + mi_heap_stat_increase(heap, page_bins[_mi_page_stats_bin(page)], 1); if (pq != NULL) { mi_page_queue_push(heap, pq, page); } mi_assert_expensive(_mi_page_is_valid(page)); return page; @@ -817,6 +817,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p } // for each page mi_heap_stat_counter_increase(heap, page_searches, count); + mi_heap_stat_counter_increase(heap, page_searches_count, 1); // set the page to the best candidate if (page_candidate != NULL) { @@ -982,9 +983,9 @@ static mi_page_t* mi_find_page(mi_heap_t* heap, size_t size, size_t huge_alignme // Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -// The `huge_alignment` is normally 0 but is set to a multiple of MI_SEGMENT_SIZE for -// very large requested alignments in which case we use a huge segment. -void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept +// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for +// very large requested alignments in which case we use a huge singleton page. +void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept { mi_assert_internal(heap != NULL); @@ -1033,12 +1034,12 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al void* p; if mi_unlikely(zero && mi_page_is_huge(page)) { // note: we cannot call _mi_page_malloc with zeroing for huge blocks; we zero it afterwards in that case. - p = _mi_page_malloc(heap, page, size); + p = _mi_page_malloc_zero(heap, page, size, false, usable); mi_assert_internal(p != NULL); _mi_memzero_aligned(p, mi_page_usable_block_size(page)); } else { - p = _mi_page_malloc_zero(heap, page, size, zero); + p = _mi_page_malloc_zero(heap, page, size, zero, usable); mi_assert_internal(p != NULL); } // move singleton pages to the full queue diff --git a/compat/mimalloc/prim/unix/prim.c b/compat/mimalloc/prim/unix/prim.c index 650aa657b9eb18..99331e3f8215c1 100644 --- a/compat/mimalloc/prim/unix/prim.c +++ b/compat/mimalloc/prim/unix/prim.c @@ -32,6 +32,7 @@ terms of the MIT license. A copy of the license can be found in the file #if defined(__linux__) #include #include // THP disable, PR_SET_VMA + #include // sysinfo #if defined(__GLIBC__) && !defined(PR_SET_VMA) #include #endif @@ -49,6 +50,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !defined(MAC_OS_X_VERSION_10_7) #define MAC_OS_X_VERSION_10_7 1070 #endif + #include #elif defined(__FreeBSD__) || defined(__DragonFly__) #include #if __FreeBSD_version >= 1200000 @@ -119,43 +121,71 @@ static inline int mi_prim_access(const char *fpath, int mode) { static bool unix_detect_overcommit(void) { bool os_overcommit = true; -#if defined(__linux__) - int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); - if (fd >= 0) { - char buf[32]; - ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); - mi_prim_close(fd); - // - // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) - if (nread >= 1) { - os_overcommit = (buf[0] == '0' || buf[0] == '1'); + #if defined(__linux__) + int fd = mi_prim_open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd >= 0) { + char buf[32]; + ssize_t nread = mi_prim_read(fd, &buf, sizeof(buf)); + mi_prim_close(fd); + // + // 0: heuristic overcommit, 1: always overcommit, 2: never overcommit (ignore NORESERVE) + if (nread >= 1) { + os_overcommit = (buf[0] == '0' || buf[0] == '1'); + } } - } -#elif defined(__FreeBSD__) - int val = 0; - size_t olen = sizeof(val); - if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { - os_overcommit = (val != 0); - } -#else - // default: overcommit is true -#endif + #elif defined(__FreeBSD__) + int val = 0; + size_t olen = sizeof(val); + if (sysctlbyname("vm.overcommit", &val, &olen, NULL, 0) == 0) { + os_overcommit = (val != 0); + } + #else + // default: overcommit is true + #endif return os_overcommit; } +// try to detect the physical memory dynamically (if possible) +static void unix_detect_physical_memory( size_t page_size, size_t* physical_memory_in_kib ) { + #if defined(CTL_HW) && (defined(HW_PHYSMEM64) || defined(HW_MEMSIZE)) // freeBSD, macOS + MI_UNUSED(page_size); + int64_t physical_memory = 0; + size_t length = sizeof(int64_t); + #if defined(HW_PHYSMEM64) + int mib[2] = { CTL_HW, HW_PHYSMEM64 }; + #else + int mib[2] = { CTL_HW, HW_MEMSIZE }; + #endif + const int err = sysctl(mib, 2, &physical_memory, &length, NULL, 0); + if (err==0 && physical_memory > 0) { + const int64_t phys_in_kib = physical_memory / MI_KiB; + if (phys_in_kib > 0 && (uint64_t)phys_in_kib <= SIZE_MAX) { + *physical_memory_in_kib = (size_t)phys_in_kib; + } + } + #elif defined(__linux__) + MI_UNUSED(page_size); + struct sysinfo info; _mi_memzero_var(info); + const int err = sysinfo(&info); + if (err==0 && info.totalram > 0 && info.totalram <= SIZE_MAX) { + *physical_memory_in_kib = (size_t)info.totalram / MI_KiB; + } + #elif defined(_SC_PHYS_PAGES) // do not use by default as it might cause allocation (by using `fopen` to parse /proc/meminfo) (issue #1100) + const long pphys = sysconf(_SC_PHYS_PAGES); + const size_t psize_in_kib = page_size / MI_KiB; + if (psize_in_kib > 0 && pphys > 0 && (unsigned long)pphys <= SIZE_MAX && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { + *physical_memory_in_kib = (size_t)pphys * psize_in_kib; + } + #endif +} + void _mi_prim_mem_init( mi_os_mem_config_t* config ) { long psize = sysconf(_SC_PAGESIZE); - if (psize > 0) { + if (psize > 0 && (unsigned long)psize < SIZE_MAX) { config->page_size = (size_t)psize; config->alloc_granularity = (size_t)psize; - #if defined(_SC_PHYS_PAGES) - long pphys = sysconf(_SC_PHYS_PAGES); - const size_t psize_in_kib = (size_t)psize / MI_KiB; - if (psize_in_kib > 0 && pphys > 0 && (size_t)pphys <= (SIZE_MAX/psize_in_kib)) { - config->physical_memory_in_kib = (size_t)pphys * psize_in_kib; - } - #endif + unix_detect_physical_memory(config->page_size, &config->physical_memory_in_kib); } config->large_page_size = MI_UNIX_LARGE_PAGE_SIZE; config->has_overcommit = unix_detect_overcommit(); @@ -167,7 +197,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) #if defined(MI_NO_THP) if (true) #else - if (!mi_option_is_enabled(mi_option_allow_large_os_pages)) // disable THP also if large OS pages are not allowed in the options + if (!mi_option_is_enabled(mi_option_allow_thp)) // disable THP if requested through an option #endif { int val = 0; @@ -295,7 +325,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif // huge page allocation - if (allow_large && (large_only || (_mi_os_use_large_page(size, try_alignment) && mi_option_get(mi_option_allow_large_os_pages) == 1))) { + if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) { static _Atomic(size_t) large_page_try_ok; // = 0; size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { @@ -354,7 +384,8 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec if (p == NULL) { *is_large = false; p = unix_mmap_prim_aligned(addr, size, try_alignment, protect_flags, flags, fd); - if (p != NULL) { + #if !defined(MI_NO_THP) + if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { #if defined(MADV_HUGEPAGE) // Many Linux systems don't allow MAP_HUGETLB but they support instead // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE @@ -362,22 +393,19 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec // in that case -- in particular for our large regions (in `memory.c`). // However, some systems only allow THP if called with explicit `madvise`, so // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (allow_large && _mi_os_use_large_page(size, try_alignment)) { - if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { - // *is_large = true; // possibly - }; - } + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + // *is_large = true; // possibly + }; #elif defined(__sun) - if (allow_large && _mi_os_use_large_page(size, try_alignment)) { - struct memcntl_mha cmd = {0}; - cmd.mha_pagesize = _mi_os_large_page_size(); - cmd.mha_cmd = MHA_MAPSIZE_VA; - if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { - // *is_large = true; // possibly - } + struct memcntl_mha cmd = {0}; + cmd.mha_pagesize = _mi_os_large_page_size(); + cmd.mha_cmd = MHA_MAPSIZE_VA; + if (memcntl((caddr_t)p, size, MC_HAT_ADVISE, (caddr_t)&cmd, 0, 0) == 0) { + // *is_large = true; // possibly } #endif } + #endif } return p; } @@ -446,7 +474,7 @@ int _mi_prim_decommit(void* start, size_t size, bool* needs_recommit) { #else // decommit: use MADV_DONTNEED as it decreases rss immediately (unlike MADV_FREE) err = unix_madvise(start, size, MADV_DONTNEED); - #endif + #endif #if !MI_DEBUG && MI_SECURE<=2 *needs_recommit = false; #else @@ -467,8 +495,8 @@ int _mi_prim_reset(void* start, size_t size) { int err = 0; // on macOS can use MADV_FREE_REUSABLE (but we disable this for now as it seems slower) - #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) - err = unix_madvise(start, size, MADV_FREE_REUSABLE); + #if 0 && defined(__APPLE__) && defined(MADV_FREE_REUSABLE) + err = unix_madvise(start, size, MADV_FREE_REUSABLE); if (err==0) return 0; // fall through #endif diff --git a/compat/mimalloc/prim/windows/prim.c b/compat/mimalloc/prim/windows/prim.c index eebdc4a67e2f9f..75a93d2a7277de 100644 --- a/compat/mimalloc/prim/windows/prim.c +++ b/compat/mimalloc/prim/windows/prim.c @@ -287,8 +287,9 @@ static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DW static _Atomic(size_t) large_page_try_ok; // = 0; void* p = NULL; // Try to allocate large OS pages (2MiB) if allowed or required. - if ((large_only || _mi_os_use_large_page(size, try_alignment)) - && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) { + if ((large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages))) + && allow_large && (flags&MEM_COMMIT)!=0 && (flags&MEM_RESERVE)!=0) + { size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); if (!large_only && try_ok > 0) { // if a large page allocation fails, it seems the calls to VirtualAlloc get very expensive. @@ -572,7 +573,7 @@ void _mi_prim_out_stderr( const char* msg ) #ifdef MI_HAS_CONSOLE_IO CONSOLE_SCREEN_BUFFER_INFO sbi; hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); - #endif + #endif } const size_t len = _mi_strlen(msg); if (len > 0 && len < UINT32_MAX) { @@ -580,7 +581,7 @@ void _mi_prim_out_stderr( const char* msg ) if (hconIsConsole) { #ifdef MI_HAS_CONSOLE_IO WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); - #endif + #endif } else if (hcon != INVALID_HANDLE_VALUE) { // use direct write if stderr was redirected diff --git a/compat/mimalloc/segment.c b/compat/mimalloc/segment.c index 32841e6deef20e..6f398822dfb421 100644 --- a/compat/mimalloc/segment.c +++ b/compat/mimalloc/segment.c @@ -1023,7 +1023,7 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld size_t inuse = page->capacity * mi_page_block_size(page); _mi_stat_decrease(&tld->stats->page_committed, inuse); _mi_stat_decrease(&tld->stats->pages, 1); - _mi_stat_decrease(&tld->stats->page_bins[_mi_page_bin(page)], 1); + _mi_stat_decrease(&tld->stats->page_bins[_mi_page_stats_bin(page)], 1); // reset the page memory to reduce memory pressure? if (segment->allow_decommit && mi_option_is_enabled(mi_option_deprecated_page_reset)) { diff --git a/compat/mimalloc/stats.c b/compat/mimalloc/stats.c index 34b3d4e4ce44c8..36e8c9813edb09 100644 --- a/compat/mimalloc/stats.c +++ b/compat/mimalloc/stats.c @@ -86,11 +86,15 @@ void _mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) { if (stat==src) return; mi_atomic_void_addi64_relaxed(&stat->total, &src->total); - mi_atomic_void_addi64_relaxed(&stat->current, &src->current); - // peak scores do really not work across threads .. we just add them - mi_atomic_void_addi64_relaxed( &stat->peak, &src->peak); - // or, take the max? - // mi_atomic_maxi64_relaxed(&stat->peak, src->peak); + const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current); + + // Global current plus thread peak approximates new global peak + // note: peak scores do really not work across threads. + // we used to just add them together but that often overestimates in practice. + // similarly, max does not seem to work well. The current approach + // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 + // for a longer description. + mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak); } static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) { @@ -212,12 +216,6 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_stat_print_ex(stat, msg, unit, out, arg, NULL); } -static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { - _mi_fprintf(out, arg, "%10s:", msg); - mi_print_amount(stat->peak, unit, out, arg); - _mi_fprintf(out, arg, "\n"); -} - #if MI_STAT>1 static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { _mi_fprintf(out, arg, "%10s:", msg); @@ -234,8 +232,8 @@ static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg } -static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { - const int64_t avg_tens = (stat->total == 0 ? 0 : (stat->total*10 / stat->total)); +static void mi_stat_average_print(size_t count, size_t total, const char* msg, mi_output_fun* out, void* arg) { + const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count)); const long avg_whole = (long)(avg_tens/10); const long avg_frac1 = (long)(avg_tens%10); _mi_fprintf(out, arg, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1); @@ -330,8 +328,8 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) #endif mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, ""); mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, ""); - mi_stat_peak_print(&stats->reset, "reset", 1, out, arg ); - mi_stat_peak_print(&stats->purged, "purged", 1, out, arg ); + mi_stat_counter_print(&stats->reset, "reset", out, arg ); + mi_stat_counter_print(&stats->purged, "purged", out, arg ); mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, ""); mi_stat_print(&stats->segments, "segments", -1, out, arg); mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg); @@ -349,7 +347,7 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_stat_counter_print(&stats->purge_calls, "purges", out, arg); mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg); mi_stat_print(&stats->threads, "threads", -1, out, arg); - mi_stat_counter_print_avg(&stats->page_searches, "searches", out, arg); + mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); _mi_fprintf(out, arg, "%10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); size_t elapsed; @@ -362,11 +360,11 @@ static void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) size_t page_faults; mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); _mi_fprintf(out, arg, "%10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); - _mi_fprintf(out, arg, "%10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, rss: ", "process", + _mi_fprintf(out, arg, "%10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults ); mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s"); if (peak_commit > 0) { - _mi_fprintf(out, arg, ", commit: "); + _mi_fprintf(out, arg, ", peak commit: "); mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); } _mi_fprintf(out, arg, "\n");