From cc019fd1450333a1b55ff3df441039fd067612d0 Mon Sep 17 00:00:00 2001 From: daan Date: Thu, 13 Feb 2020 18:57:12 -0800 Subject: [PATCH] improve inlining and codegen for mi_new --- include/mimalloc-internal.h | 52 ++++++++++++++++++------------------- include/mimalloc.h | 24 ++++++++--------- src/alloc.c | 8 +++--- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index cd49e39e..e4da1124 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -290,13 +290,13 @@ mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing hea #if defined(MI_MALLOC_OVERRIDE) #if defined(__MACH__) // OSX -#define MI_TLS_SLOT 89 // seems unused? +#define MI_TLS_SLOT 89 // seems unused? // other possible unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89) // see #elif defined(__OpenBSD__) -// use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) +// use end bytes of a name; goes wrong if anyone uses names > 23 characters (ptrhread specifies 16) // see -#define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) +#define MI_TLS_PTHREAD_SLOT_OFS (6*sizeof(int) + 4*sizeof(void*) + 24) #elif defined(__DragonFly__) #warning "mimalloc is not working correctly on DragonFly yet." #define MI_TLS_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?) @@ -754,89 +754,89 @@ static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { // implementations for the first 4 versions so only the 5th needs to be implemented. // ------------------------------------------------------------------------------------------------------------- #ifdef NDEBUG -#define MI_DEBUG_ONLY(x) +#define MI_DEBUG_ONLY(x) #else #define MI_DEBUG_ONLY(x) x #endif #define MI_ALLOC_API1(tp,name,tp0,arg0,tp1,arg1) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + static tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept; \ tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XRET()); } \ tp mi_heap_##name(mi_heap_t* heap, tp1 arg1) mi_attr_noexcept { return mi_base_##name(heap, arg1 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1 MI_SOURCE_XARG); }) \ MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept + static tp mi_base_##name(tp0 arg0, tp1 arg1 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API2(tp,name,tp0,arg0,tp1,arg1,tp2,arg2) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept; \ tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XRET()); } \ tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2 MI_SOURCE_XARG); }) \ MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API3(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept; \ tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XRET()); } \ tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3 MI_SOURCE_XARG); }) \ MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API4(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3,tp4,arg4) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept; \ tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_ALLOC_API5(tp,name,tp0,arg0,tp1,arg1,tp2,arg2,tp3,arg3,tp4,arg4,tp5,arg5) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept; \ + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept; \ tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ tp mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5) mi_attr_noexcept { return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(mi_get_default_heap(), arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); }) \ MI_DEBUG_ONLY(tp dbg_mi_heap_##name(mi_heap_t* heap, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(heap, arg1, arg2, arg3, arg4, arg5 MI_SOURCE_XARG); } ) \ - static inline tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept + static tp mi_base_##name(tp0 arg0, tp1 arg1, tp2 arg2, tp3 arg3, tp4 arg4, tp5 arg5 MI_SOURCE_XPARAM) mi_attr_noexcept #define MI_NEW_API1(tp,name,tp1,arg1) \ - static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ + static tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ tp mi_##name(tp1 arg1) { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) + static tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) #define MI_NEW_API2(tp,name,tp1,arg1,tp2,arg2) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ + static tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ tp mi_##name(tp1 arg1, tp2 arg2) { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) + static tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) #define MI_NEW_API3(tp,name,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ + static tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) + static tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) #define MI_SOURCE_API1(tp,name,tp1,arg1) \ - static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ + static tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM); \ tp mi_##name(tp1 arg1) mi_attr_noexcept { return mi_base_##name(arg1 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) + static tp mi_base_##name(tp1 arg1 MI_SOURCE_XPARAM) #define MI_SOURCE_API2(tp,name,tp1,arg1,tp2,arg2) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ + static tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM); \ tp mi_##name(tp1 arg1, tp2 arg2) mi_attr_noexcept { return mi_base_##name(arg1, arg2 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) + static tp mi_base_##name(tp1 arg1, tp2 arg2 MI_SOURCE_XPARAM) #define MI_SOURCE_API3(tp,name,tp1,arg1,tp2,arg2,tp3,arg3) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ + static tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM); \ tp mi_##name(tp1 arg1, tp2 arg2, tp3 arg3) mi_attr_noexcept { return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XRET()); } \ MI_DEBUG_ONLY(tp dbg_mi_##name(tp1 arg1, tp2 arg2, tp3 arg3, mi_source_t __mi_source) mi_attr_noexcept { (void)__mi_source; return mi_base_##name(arg1, arg2, arg3 MI_SOURCE_XARG); }) \ - static inline tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) + static tp mi_base_##name(tp1 arg1, tp2 arg2, tp3 arg3 MI_SOURCE_XPARAM) #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index 9b161ab8..94df477d 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -528,18 +528,18 @@ template bool operator!=(const mi_stl_allocator&, const #define mi_rezalloc_aligned_at(p,n,a,o) MI_SOURCE_LOC(mi_rezalloc_aligned_at,p,n,a,o) #define mi_recalloc_aligned_at(p,c,n,a,o) MI_SOURCE_LOC(mi_recalloc_aligned_at,p,c,n,a,o) -#define mi_heap_malloc(h,n) MI_SOURCE_LOC(mi_heap_malloc,h,n) -#define mi_heap_calloc(h,c,n) MI_SOURCE_LOC(mi_heap_calloc,h,c,n) -#define mi_heap_realloc(h,p,n) MI_SOURCE_LOC(mi_heap_realloc,h,p,n) -#define mi_heap_strdup(h,s) MI_SOURCE_LOC(mi_heap_strdup,h,s) -#define mi_heap_strndup(h,s,n) MI_SOURCE_LOC(mi_heap_strndup,h,s,n) -#define mi_heap_realpath(h,f,n) MI_SOURCE_LOC(mi_heap_realpath,h,f,n) -#define mi_heap_malloc_small(h,n) MI_SOURCE_LOC(mi_heap_malloc_small,h,n) -#define mi_heap_zalloc_small(h,n) MI_SOURCE_LOC(mi_heap_zalloc_small,h,n) -#define mi_heap_zalloc(h,n) MI_SOURCE_LOC(mi_heap_zalloc,h,n) -#define mi_heap_mallocn(h,c,n) MI_SOURCE_LOC(mi_heap_mallocn,h,c,n) -#define mi_heap_reallocn(h,p,c,n) MI_SOURCE_LOC(mi_heap_reallocn,h,p,c,n) -#define mi_heap_reallocf(h,p,n) MI_SOURCE_LOC(mi_heap_reallocf,h,p,n) +#define mi_heap_malloc(h,n) MI_SOURCE_LOC(mi_heap_malloc,h,n) +#define mi_heap_calloc(h,c,n) MI_SOURCE_LOC(mi_heap_calloc,h,c,n) +#define mi_heap_realloc(h,p,n) MI_SOURCE_LOC(mi_heap_realloc,h,p,n) +#define mi_heap_strdup(h,s) MI_SOURCE_LOC(mi_heap_strdup,h,s) +#define mi_heap_strndup(h,s,n) MI_SOURCE_LOC(mi_heap_strndup,h,s,n) +#define mi_heap_realpath(h,f,n) MI_SOURCE_LOC(mi_heap_realpath,h,f,n) +#define mi_heap_malloc_small(h,n) MI_SOURCE_LOC(mi_heap_malloc_small,h,n) +#define mi_heap_zalloc_small(h,n) MI_SOURCE_LOC(mi_heap_zalloc_small,h,n) +#define mi_heap_zalloc(h,n) MI_SOURCE_LOC(mi_heap_zalloc,h,n) +#define mi_heap_mallocn(h,c,n) MI_SOURCE_LOC(mi_heap_mallocn,h,c,n) +#define mi_heap_reallocn(h,p,c,n) MI_SOURCE_LOC(mi_heap_reallocn,h,p,c,n) +#define mi_heap_reallocf(h,p,n) MI_SOURCE_LOC(mi_heap_reallocf,h,p,n) #define mi_heap_malloc_aligned(h,n,a) MI_SOURCE_LOC(mi_heap_malloc_aligned,h,n,a) #define mi_heap_zalloc_aligned(h,n,a) MI_SOURCE_LOC(mi_heap_zalloc_aligned,h,n,a) diff --git a/src/alloc.c b/src/alloc.c index bb1c884c..20baa5b9 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -62,7 +62,7 @@ extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t siz } // allocate a small block -MI_ALLOC_API1(mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, size_t, size) +MI_ALLOC_API1(inline mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, size_t, size) { mi_assert(heap!=NULL); mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local @@ -81,7 +81,7 @@ MI_ALLOC_API1(mi_decl_restrict void*, malloc_small, mi_heap_t*, heap, size_t, si // The main allocation function -MI_ALLOC_API1(mi_decl_restrict void*, malloc, mi_heap_t*, heap, size_t, size) +MI_ALLOC_API1(inline mi_decl_restrict void*, malloc, mi_heap_t*, heap, size_t, size) { if (mi_likely(size <= MI_SMALL_SIZE_MAX)) { return mi_base_malloc_small(heap, size MI_SOURCE_XARG); @@ -398,7 +398,7 @@ void mi_free(void* p) mi_attr_noexcept "(this may still be a valid very large allocation (over 64MiB))\n", p); if (mi_likely(_mi_ptr_cookie(segment) == segment->cookie)) { _mi_warning_message("(yes, the previous pointer %p was valid after all)\n", p); - } + } } #endif #if (MI_DEBUG!=0 || MI_SECURE>=4) @@ -537,7 +537,7 @@ MI_ALLOC_API2(mi_decl_restrict void*, mallocn, mi_heap_t*, heap, size_t, count, // Expand in place or fail -MI_ALLOC_API2(void*, expand, mi_heap_t*, heap, void*, p, size_t, newsize) +MI_ALLOC_API2(void*, expand, mi_heap_t*, heap, void*, p, size_t, newsize) { UNUSED(heap); #ifndef NDEBUG