diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index feb3abae..8f56945b 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -32,26 +32,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_cache_align #endif -/* ----------------------------------------------------------- - Padding ------------------------------------------------------------ */ -#if (MI_PADDING) -#define MI_EXTRA_PADDING_XPARAM , size_t __extra_padding -#define MI_EXTRA_PADDING_XARG , __extra_padding -#define MI_EXTRA_PADDING_ARG __extra_padding -static inline size_t mi_extra_padding() { - return MI_PADDING_SIZE + mi_option_get(mi_option_debug_extra_padding); -} -#else -#define MI_EXTRA_PADDING_XPARAM -#define MI_EXTRA_PADDING_XARG -#define MI_EXTRA_PADDING_ARG 0 -static inline size_t mi_extra_padding() { - return 0; -} -#endif - - // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); @@ -110,7 +90,7 @@ void _mi_abandoned_await_readers(void); // "page.c" -void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM MI_SOURCE_XPARAM) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -143,7 +123,7 @@ mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); // "alloc.c" -void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_EXTRA_PADDING_XPARAM MI_SOURCE_XPARAM) mi_attr_noexcept; // called from `_mi_malloc_generic` +void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept; // called from `_mi_malloc_generic` mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); bool _mi_free_delayed_block(mi_block_t* block); void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size); @@ -158,6 +138,21 @@ bool _mi_page_is_valid(mi_page_t* page); #endif +/* ----------------------------------------------------------- + Padding +----------------------------------------------------------- */ +#if (MI_PADDING) +static inline size_t mi_extra_padding(mi_heap_t* const heap) { + return (MI_PADDING_SIZE + heap->extra_padding); +} +#else +static inline size_t mi_extra_padding() { + return 0; +} +#endif + + + // ------------------------------------------------------ // Branches // ------------------------------------------------------ diff --git a/include/mimalloc-types.h b/include/mimalloc-types.h index ca72aceb..df842028 100644 --- a/include/mimalloc-types.h +++ b/include/mimalloc-types.h @@ -334,6 +334,7 @@ struct mi_heap_s { size_t page_count; // total number of pages in the `pages` queues. size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) size_t page_retired_max; // largest retired index into the `pages` array. + size_t extra_padding; // extra padding bytes in each heap block to better detect heap block overflows mi_heap_t* next; // list of heaps per thread bool no_reclaim; // `true` if this heap should not reclaim abandoned pages }; diff --git a/include/mimalloc.h b/include/mimalloc.h index 7ffb55d9..6b346daa 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -296,11 +296,12 @@ typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_ mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); +// Experimental mi_decl_export void mi_heap_print_json(mi_heap_t* heap, mi_output_fun* out, void* arg); mi_decl_export bool mi_heap_is_empty(mi_heap_t* heap); mi_decl_export void mi_heap_check_leak(mi_heap_t* heap, mi_output_fun* out, void* arg); +mi_decl_export void mi_heap_set_extra_padding(mi_heap_t* heap, size_t extra_padding); -// Experimental mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export bool mi_is_redirected() mi_attr_noexcept; diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 14ed76c6..3623c3ed 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -24,17 +24,17 @@ static mi_decl_restrict void* mi_base_malloc_zero_aligned_at(mi_heap_t* const he const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` // try if there is a small block available with just the right alignment - const size_t __extra_padding = mi_extra_padding(); - const size_t padsize = size + __extra_padding; // safe for overflow as size <= PTRDIFF_MAX + const size_t extra_padding = mi_extra_padding(heap); + const size_t padsize = size + extra_padding; // safe for overflow as size <= PTRDIFF_MAX if (mi_likely(padsize <= MI_SMALL_SIZE_MAX)) { mi_page_t* page = _mi_heap_get_free_small_page(heap,padsize); const bool is_aligned = (((uintptr_t)page->free+offset) & align_mask)==0; if (mi_likely(page->free != NULL && is_aligned)) { #if MI_STAT>1 - mi_heap_stat_increase( heap, malloc, size); + mi_heap_stat_increase( heap, malloc, padsize); #endif - void* p = _mi_page_malloc(heap,page,padsize MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); // TODO: inline _mi_page_malloc + void* p = _mi_page_malloc(heap,page,padsize MI_SOURCE_XARG); // TODO: inline _mi_page_malloc mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); if (zero) _mi_block_zero_init(page,p,size); @@ -44,7 +44,7 @@ static mi_decl_restrict void* mi_base_malloc_zero_aligned_at(mi_heap_t* const he // use regular allocation if it is guaranteed to fit the alignment constraints if (offset==0 && alignment<=padsize && padsize<=MI_MEDIUM_OBJ_SIZE_MAX && (padsize&align_mask)==0) { - void* p = _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG); + void* p = _mi_base_malloc_zero(heap, size, zero MI_SOURCE_XARG); // base malloc adds padding again to size mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); return p; } diff --git a/src/alloc.c b/src/alloc.c index bbededcb..dce834b1 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -28,11 +28,11 @@ terms of the MIT license. A copy of the license can be found in the file // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. -extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_EXTRA_PADDING_XPARAM MI_SOURCE_XPARAM) mi_attr_noexcept { +extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert_internal(page->xblock_size==0||mi_page_block_size(page) >= size); mi_block_t* block = page->free; if (mi_unlikely(block == NULL)) { - return _mi_malloc_generic(heap, size MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); // slow path + return _mi_malloc_generic(heap, size MI_SOURCE_XARG); // slow path } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); // pop from the free list @@ -52,9 +52,11 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } #endif #if (MI_PADDING>0) && defined(MI_ENCODE_FREELIST) + const size_t extra_padding = mi_extra_padding(heap); + mi_assert_internal(extra_padding <= size && extra_padding >= MI_PADDING_SIZE); mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + mi_page_usable_block_size(page)); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - __extra_padding)); - mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - __extra_padding + delta)); + ptrdiff_t delta = extra_padding - MI_PADDING_SIZE; + mi_assert_internal(delta >= 0 && mi_page_usable_block_size(page) >= (size - extra_padding + delta)); padding->canary = (uint32_t)(mi_ptr_encode(page,block,page->keys)); padding->delta = (uint32_t)(delta); padding->source = __mi_source; @@ -71,17 +73,17 @@ MI_ALLOC_API1(inline mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, siz mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local mi_assert(size <= MI_SMALL_SIZE_MAX); - const size_t __extra_padding = mi_extra_padding(); + const size_t extra_padding = mi_extra_padding(heap); #if (MI_PADDING) if (size == 0) { size = sizeof(void*); } - if ((size + __extra_padding) > MI_SMALL_SIZE_MAX) { + if ((size + extra_padding) > MI_SMALL_SIZE_MAX) { return MI_SOURCE_ARG(mi_heap_malloc, heap, size); // call base malloc in case we were invoked directly } #endif - mi_page_t* page = _mi_heap_get_free_small_page(heap,size + __extra_padding); - void* p = _mi_page_malloc(heap, page, size + __extra_padding MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); + mi_page_t* page = _mi_heap_get_free_small_page(heap,size + extra_padding); + void* p = _mi_page_malloc(heap, page, size + extra_padding MI_SOURCE_XARG); mi_assert_internal(p==NULL || mi_usable_size(p) >= size); #if MI_STAT>1 if (p != NULL) { @@ -96,14 +98,14 @@ MI_ALLOC_API1(inline mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, siz // The main allocation function MI_ALLOC_API1(inline mi_decl_restrict void*, malloc, mi_heap_t*, heap, size_t, size) { - const size_t __extra_padding = mi_extra_padding(); - if (mi_likely(size <= MI_SMALL_SIZE_MAX - __extra_padding && __extra_padding < MI_SMALL_SIZE_MAX)) { // careful for overflow + const size_t extra_padding = mi_extra_padding(heap); + if (mi_likely(size <= MI_SMALL_SIZE_MAX - extra_padding && extra_padding < MI_SMALL_SIZE_MAX)) { // careful for overflow return mi_base_malloc_small(heap, size MI_SOURCE_XARG); } else { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local - void* const p = _mi_malloc_generic(heap, size + __extra_padding MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); // note: size + __extra_padding can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(heap, size + extra_padding MI_SOURCE_XARG); // note: size + __extra_padding can overflow but it is detected in malloc_generic mi_assert_internal(p == NULL || mi_usable_size(p) >= size); #if MI_STAT>1 if (p != NULL) { diff --git a/src/heap.c b/src/heap.c index 17498ba2..e2ad1466 100644 --- a/src/heap.c +++ b/src/heap.c @@ -200,6 +200,7 @@ mi_heap_t* mi_heap_new(void) { heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); + heap->extra_padding = bheap->extra_padding; heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe // push on the thread local heaps list heap->next = heap->tld->heaps; @@ -211,6 +212,11 @@ uintptr_t _mi_heap_random_next(mi_heap_t* heap) { return _mi_random_next(&heap->random); } +void mi_heap_set_extra_padding(mi_heap_t* heap, size_t extra_padding) { + if (extra_padding > 1*MiB) extra_padding = 1*MiB; + heap->extra_padding = extra_padding; +} + // zero out the page queues static void mi_heap_reset_pages(mi_heap_t* heap) { mi_assert_internal(mi_heap_is_initialized(heap)); @@ -580,13 +586,17 @@ static bool mi_heap_print_json_visit(const mi_heap_t* heap, const mi_heap_area_t _mi_fprintf(varg->out, varg->out_arg, varg->area_count==0 ? " {" : " ]\n}\n,{"); varg->area_count++; varg->block_count = 0; - _mi_fprintf(varg->out, varg->out_arg, "\"page\": %zu, \"start\": 0x%p, \"block_size\": %zu, \"used_size\": %zu,\n \"reserved\": %zu, \"committed\": %zu,", varg->area_count, area->blocks, area->block_size, area->used, area->reserved, area->committed); + _mi_fprintf(varg->out, varg->out_arg, + "\"page\": %zu, \"start\": 0x%p, \"block_size\": %zu, \"used_size\": %zu,\n \"reserved\": %zu, \"committed\": %zu,", + varg->area_count, area->blocks, area->block_size, area->used, area->reserved, area->committed); _mi_fprintf(varg->out, varg->out_arg, " \"blocks\": [\n"); } else { _mi_fprintf(varg->out, varg->out_arg, varg->block_count==0 ? " {" : " ,{"); varg->block_count++; - _mi_fprintf(varg->out, varg->out_arg, "\"block\": 0x%p, \"valid\": %s, \"size\": %zu, \"usable_size\": %zu, \"allocated_size\": %zu,\n ", info->block, info->valid ? "true" : "false", info->size, info->usable_size, info->allocated_size); + _mi_fprintf(varg->out, varg->out_arg, + "\"block\": 0x%p, \"valid\": %s, \"size\": %zu, \"usable_size\": %zu, \"allocated_size\": %zu,\n ", + info->block, info->valid ? "true" : "false", info->size, info->usable_size, info->allocated_size); int lineno; const char* fname; void* ret = mi_source_unpack(info->source, &fname, &lineno); @@ -600,7 +610,7 @@ static bool mi_heap_print_json_visit(const mi_heap_t* heap, const mi_heap_area_t void mi_heap_print_json(mi_heap_t* heap, mi_output_fun* out, void* arg) { if (heap==NULL) heap = mi_heap_get_default(); mi_print_json_t info = { 0, 0, out, arg }; - _mi_fprintf(info.out, info.out_arg, "{ \"heap\": 0x%p, \"thread_id\": 0x%zx, \"page_count\": %zu, \"block_padding\": %zu", heap, heap->thread_id, heap->page_count, mi_extra_padding() ); + _mi_fprintf(info.out, info.out_arg, "{ \"heap\": 0x%p, \"thread_id\": 0x%zx, \"page_count\": %zu, \"block_padding\": %zu", heap, heap->thread_id, heap->page_count, mi_extra_padding(heap) ); _mi_fprintf(info.out, info.out_arg, ", \"pages\": [\n"); mi_heap_visit_blocks(heap, true, &mi_heap_print_json_visit, &info); _mi_fprintf(info.out, info.out_arg, info.area_count==0 ? "]\n" : " ] }\n] }\n"); diff --git a/src/init.c b/src/init.c index f5ed3014..b4c0e757 100644 --- a/src/init.c +++ b/src/init.c @@ -98,6 +98,7 @@ const mi_heap_t _mi_heap_empty = { { {0}, {0}, 0 }, 0, // page count MI_BIN_FULL, 0, // page retired min/max + 0, // extra padding NULL, // next false }; @@ -133,6 +134,7 @@ mi_heap_t _mi_heap_main = { { {0x846ca68b}, {0}, 0 }, // random 0, // page count MI_BIN_FULL, 0, // page retired min/max + 0, // extra_padding NULL, // next heap false // can reclaim }; @@ -149,6 +151,7 @@ static void mi_heap_main_init(void) { _mi_random_init(&_mi_heap_main.random); _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); + _mi_heap_main.extra_padding = mi_option_get(mi_option_debug_extra_padding); } } @@ -195,6 +198,7 @@ static bool _mi_heap_init(void) { heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); heap->tld = tld; + heap->extra_padding = _mi_heap_main.extra_padding; tld->heap_backing = heap; tld->heaps = heap; tld->segments.stats = &tld->stats; diff --git a/src/page.c b/src/page.c index 010769b7..b7293d43 100644 --- a/src/page.c +++ b/src/page.c @@ -792,7 +792,7 @@ static mi_page_t* mi_huge_page_alloc(mi_heap_t* heap, size_t size) { // Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM MI_SOURCE_XPARAM) mi_attr_noexcept +void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_SOURCE_XPARAM) mi_attr_noexcept { mi_assert_internal(heap != NULL); @@ -811,8 +811,8 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM // huge allocation? mi_page_t* page; - const size_t req_size = size - MI_EXTRA_PADDING_ARG; // correct for padding_size in case of an overflow on `size` - if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - MI_EXTRA_PADDING_ARG) )) { + const size_t req_size = size - mi_extra_padding(heap); // correct for padding_size in case of an overflow on `size` + if (mi_unlikely(req_size > (MI_LARGE_OBJ_SIZE_MAX - mi_extra_padding(heap)) )) { if (mi_unlikely(req_size > PTRDIFF_MAX)) { // we don't allocate more than PTRDIFF_MAX (see ) _mi_error_message(EOVERFLOW, "allocation request is too large (%zu b requested)\n", req_size); return NULL; @@ -823,7 +823,7 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM } else { // otherwise find a page with free blocks in our size segregated queues - mi_assert_internal(size >= MI_EXTRA_PADDING_ARG); + mi_assert_internal(size >= mi_extra_padding(heap)); page = mi_find_free_page(heap,size); } if (mi_unlikely(page == NULL)) { // out of memory @@ -835,5 +835,5 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size MI_EXTRA_PADDING_XPARAM mi_assert_internal(mi_page_block_size(page) >= size); // and try again, this time succeeding! (i.e. this should never recurse) - return _mi_page_malloc(heap, page, size MI_EXTRA_PADDING_XARG MI_SOURCE_XARG); + return _mi_page_malloc(heap, page, size MI_SOURCE_XARG); } diff --git a/test/main-override-static.c b/test/main-override-static.c index 798b5c5a..f72ef12c 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -16,7 +16,7 @@ static void dangling_ptr_write(); int main() { mi_version(); - + mi_heap_set_extra_padding(mi_heap_get_default(), 200); // detect double frees and heap corruption // double_free1(); // double_free2(); diff --git a/test/test-api.c b/test/test-api.c index 0b911b82..093744e2 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -153,7 +153,10 @@ int main() { result = ok; }); CHECK_BODY("malloc-aligned5", { - void* p = mi_malloc_aligned(4097,4096); size_t usable = mi_usable_size(p); result = usable >= 4097 && usable < 10000; mi_free(p); + void* p = mi_malloc_aligned(4097,4096); + size_t usable = mi_usable_size(p); + result = (usable >= 4097 && usable < 12000 && ((uintptr_t)p % 4096) == 0); + mi_free(p); }); CHECK_BODY("malloc-aligned-at1", { void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p);