From 07ae64bd81bc411d740e93f8f98551150e769d1f Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Sun, 24 Mar 2024 10:39:22 -0700 Subject: [PATCH] faster check for NULL in mi_free by combining with masking --- CMakeLists.txt | 21 ++++++++++++++++++--- include/mimalloc/internal.h | 11 +++++++++-- src/free.c | 13 +++++++------ 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cc7e575..7c0f67af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,17 @@ endif() # Process options # ----------------------------------------------------------------------------- +# put -Wall early so other warnings can be disabled selectively +if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") + list(APPEND mi_cflags -Wall -Wextra -Wpedantic) +endif() +if(CMAKE_C_COMPILER_ID MATCHES "GNU") + list(APPEND mi_cflags -Wall -Wextra) +endif() +if(CMAKE_C_COMPILER_ID MATCHES "Intel") + list(APPEND mi_cflags -Wall) +endif() + if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel") set(MI_USE_CXX "ON") endif() @@ -186,6 +197,10 @@ endif() if(MI_SEE_ASM) message(STATUS "Generate assembly listings (MI_SEE_ASM=ON)") list(APPEND mi_cflags -save-temps) + if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") + message(STATUS "No GNU Line marker") + list(APPEND mi_cflags -Wno-gnu-line-marker) + endif() endif() if(MI_CHECK_FULL) @@ -279,17 +294,17 @@ endif() # Compiler flags if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU") - list(APPEND mi_cflags -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden) + list(APPEND mi_cflags -Wno-unknown-pragmas -fvisibility=hidden) if(NOT MI_USE_CXX) list(APPEND mi_cflags -Wstrict-prototypes) endif() if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") - list(APPEND mi_cflags -Wpedantic -Wno-static-in-inline) + list(APPEND mi_cflags -Wno-static-in-inline) endif() endif() if(CMAKE_C_COMPILER_ID MATCHES "Intel") - list(APPEND mi_cflags -Wall -fvisibility=hidden) + list(APPEND mi_cflags -fvisibility=hidden) endif() if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 3aad1ba4..02200594 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -416,13 +416,19 @@ static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t si // Large aligned blocks may be aligned at N*MI_SEGMENT_SIZE (inside a huge segment > MI_SEGMENT_SIZE), // and we need align "down" to the segment info which is `MI_SEGMENT_SIZE` bytes before it; // therefore we align one byte before `p`. +// We check for NULL afterwards on 64-bit systems to improve codegen for `mi_free`. static inline mi_segment_t* _mi_ptr_segment(const void* p) { - mi_assert_internal(p != NULL); - return (mi_segment_t*)(((uintptr_t)p - 1) & ~MI_SEGMENT_MASK); + mi_segment_t* const segment = (mi_segment_t*)(((uintptr_t)p - 1) & ~MI_SEGMENT_MASK); + #if MI_INTPTR_SIZE <= 4 + return (p==NULL ? NULL : segment); + #else + return ((intptr_t)segment <= 0 ? NULL : segment); + #endif } // Segment belonging to a page static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { + mi_assert_internal(page!=NULL); mi_segment_t* segment = _mi_ptr_segment(page); mi_assert_internal(segment == NULL || page == &segment->pages[page->segment_idx]); return segment; @@ -454,6 +460,7 @@ static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page // Get the page containing the pointer static inline mi_page_t* _mi_ptr_page(void* p) { + mi_assert_internal(p!=NULL); return _mi_segment_page_of(_mi_ptr_segment(p), p); } diff --git a/src/free.c b/src/free.c index 7a5a7806..0e560e53 100644 --- a/src/free.c +++ b/src/free.c @@ -98,7 +98,6 @@ void mi_decl_noinline _mi_free_generic(mi_segment_t* segment, mi_page_t* page, b static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* msg) { MI_UNUSED(msg); - mi_assert(p != NULL); #if (MI_DEBUG>0) if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) { @@ -108,7 +107,7 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms #endif mi_segment_t* const segment = _mi_ptr_segment(p); - mi_assert_internal(segment != NULL); + if mi_unlikely(segment==NULL) return segment; #if (MI_DEBUG>0) if mi_unlikely(!mi_is_in_heap_region(p)) { @@ -133,10 +132,11 @@ static inline mi_segment_t* mi_checked_ptr_segment(const void* p, const char* ms // Fast path written carefully to prevent register spilling on the stack void mi_free(void* p) mi_attr_noexcept { - if mi_unlikely(p == NULL) return; mi_segment_t* const segment = mi_checked_ptr_segment(p,"mi_free"); - const bool is_local= (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); - mi_page_t* const page = _mi_segment_page_of(segment, p); + if mi_unlikely(segment==NULL) return; + + const bool is_local = (_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); + mi_page_t* const page = _mi_segment_page_of(segment, p); if mi_likely(is_local) { // thread-local free? if mi_likely(page->flags.full_aligned == 0) { // and it is not a full page (full pages need to move from the full bin), nor has aligned blocks (aligned blocks need to be unaligned) @@ -158,6 +158,7 @@ void mi_free(void* p) mi_attr_noexcept // return true if successful bool _mi_free_delayed_block(mi_block_t* block) { // get segment and page + mi_assert_internal(block!=NULL); const mi_segment_t* const segment = _mi_ptr_segment(block); mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); mi_assert_internal(_mi_thread_id() == segment->thread_id); @@ -302,8 +303,8 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_segment_t } static inline size_t _mi_usable_size(const void* p, const char* msg) mi_attr_noexcept { - if (p == NULL) return 0; const mi_segment_t* const segment = mi_checked_ptr_segment(p, msg); + if mi_unlikely(segment==NULL) return 0; const mi_page_t* const page = _mi_segment_page_of(segment, p); if mi_likely(!mi_page_has_aligned(page)) { const mi_block_t* block = (const mi_block_t*)p;