From 461b8aed804a48006cdcf651e50660e21d8c5751 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Jul 2019 19:56:33 -0700 Subject: [PATCH 1/2] more complete C++ support for compliance and performance --- include/mimalloc.h | 14 +++-- src/alloc-override.c | 72 +++++++++++++------------ src/alloc.c | 117 +++++++++++++++++++++++++++++------------ test/main-override.cpp | 4 ++ 4 files changed, 132 insertions(+), 75 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 0fe5f2a9..ec09f119 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -259,17 +259,15 @@ mi_decl_export mi_decl_allocator void* mi_reallocarray(void* p, size_t count, si mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; + +mi_decl_export void* mi_new(size_t n) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_export void* mi_new_aligned(size_t n, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_export void* mi_new_nothrow(size_t n) mi_attr_malloc mi_attr_alloc_size(1); +mi_decl_export void* mi_new_aligned_nothrow(size_t n, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1); + #ifdef __cplusplus } #endif -#ifdef __cplusplus -#include -mi_decl_export void* mi_new(std::size_t n) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1); -#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) -#include -mi_decl_export void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) mi_attr_malloc mi_attr_alloc_size(1); -#endif -#endif #endif diff --git a/src/alloc-override.c b/src/alloc-override.c index afc36a4f..5ca88af7 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -88,60 +88,66 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); - void* operator new(std::size_t n) noexcept(false) { return mi_new(n); } - void* operator new[](std::size_t n) noexcept(false) { return mi_new(n); } + void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); + void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); - void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n); - void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept MI_FORWARD1(mi_malloc, n); + void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } + void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } #if (__cplusplus >= 201402L) - void operator delete (void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz); - void operator delete[](void* p, std::size_t sz) MI_FORWARD02(mi_free_size,p,sz); + void operator delete (void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); + void operator delete[](void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); #endif #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } - void operator delete (void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast(al)); }; - void operator delete[](void* p, std::size_t sz, std::align_val_t al) noexcept { mi_free_size_aligned(p, sz, static_cast(al)); }; + void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; + void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; - void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); } - void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n,al); } - void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast(al)); } - void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_malloc_aligned(n, static_cast(al)); } + void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } + void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } + void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } #endif -#else - // ------------------------------------------------------ - // With a C compiler we cannot override the new/delete operators - // as the standard requires calling into `get_new_handler` and/or - // throwing C++ exceptions (and we cannot do that from C). So, we - // hope the standard new uses `malloc` internally which will be - // redirected anyways. - // ------------------------------------------------------ - - #if 0 +#elif (defined(__GNUC__) || defined(__clang__)) // ------------------------------------------------------ // Override by defining the mangled C++ names of the operators (as // used by GCC and CLang). // See // ------------------------------------------------------ - void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete - void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[] + void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete + void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[] + void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); + void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); + void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } + void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } + + typedef struct mi_nothrow_s { } mi_nothrow_t; #if (MI_INTPTR_SIZE==8) - void* _Znwm(uint64_t n) MI_FORWARD1(mi_malloc,n); // new 64-bit - void* _Znam(uint64_t n) MI_FORWARD1(mi_malloc,n); // new[] 64-bit - void* _Znwmm(uint64_t n, uint64_t align) { return mi_malloc_aligned(n,align); } // aligned new 64-bit - void* _Znamm(uint64_t n, uint64_t align) { return mi_malloc_aligned(n,align); } // aligned new[] 64-bit + void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit + void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit + void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #elif (MI_INTPTR_SIZE==4) - void* _Znwj(uint32_t n) MI_FORWARD1(mi_malloc,n); // new 32-bit - void* _Znaj(uint32_t n) MI_FORWARD1(mi_malloc,n); // new[] 32-bit - void* _Znwjj(uint32_t n, uint32_t align) { return mi_malloc_aligned(n,align); } // aligned new 32-bit - void* _Znajj(uint32_t n, uint32_t align) { return mi_malloc_aligned(n,align); } // aligned new[] 32-bit + void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit + void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit + void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); + void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } + void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } + void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } #else #error "define overloads for new/delete for this platform (just for performance, can be skipped)" #endif - #endif #endif // __cplusplus diff --git a/src/alloc.c b/src/alloc.c index 1c391e64..d5050b03 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -256,8 +256,8 @@ bool _mi_free_delayed_block(mi_block_t* block) { mi_assert_internal(_mi_thread_id() == segment->thread_id); mi_page_t* page = _mi_segment_page_of(segment, block); if (mi_tf_delayed(page->thread_free) == MI_DELAYED_FREEING) { - // we might already start delayed freeing while another thread has not yet - // reset the delayed_freeing flag; in that case don't free it quite yet if + // we might already start delayed freeing while another thread has not yet + // reset the delayed_freeing flag; in that case don't free it quite yet if // this is the last block remaining. if (page->used - page->thread_freed == 1) return false; } @@ -282,9 +282,6 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept { } - - - // ------------------------------------------------------ // ensure explicit external inline definitions are emitted! // ------------------------------------------------------ @@ -298,8 +295,6 @@ void* _mi_externs[] = { (void*)&mi_heap_zalloc, (void*)&mi_heap_malloc_small }; - - #endif @@ -360,7 +355,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) if (newsize <= size && newsize >= (size / 2)) { return p; // reallocation still fits and not more than 50% waste } - void* newp = mi_heap_malloc(heap,newsize); + void* newp = mi_heap_malloc(heap,newsize); if (mi_likely(newp != NULL)) { if (zero && newsize > size) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized @@ -410,6 +405,10 @@ void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { return mi_heap_reallocf(mi_get_default_heap(),p,newsize); } +// ------------------------------------------------------ +// strdup, strndup, and realpath +// ------------------------------------------------------ + // `strdup` using mi_malloc char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { if (s == NULL) return NULL; @@ -496,41 +495,91 @@ char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name); } +/*------------------------------------------------------- +C++ new and new_aligned +The standard requires calling into `get_new_handler` and +throwing the bad_alloc exception on failure. If we compile +with a C++ compiler we can implement this precisely. If we +use a C compiler we cannot throw a `bad_alloc` exception +but we call `exit` instead (i.e. not returning). +-------------------------------------------------------*/ #ifdef __cplusplus #include - -static mi_decl_noinline void* mi_new_try(std::size_t n) noexcept(false) { - void* p; - do { - std::new_handler h = std::get_new_handler(); - if (h==NULL) throw std::bad_alloc(); +static bool mi_try_new_handler(bool nothrow) { + std::new_handler h = std::get_new_handler(); + if (h==NULL) { + if (!nothrow) throw std::bad_alloc(); + return false; + } + else { h(); - // and try again + return true; + } +} +#else +#include +#ifndef ENOMEM +#define ENOMEM 12 +#endif +typedef void (*std_new_handler_t)(); + +#if (defined(__GNUC__) || defined(__clang__)) +std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() { + return NULL; +} +std_new_handler_t mi_get_new_handler() { + return _ZSt15get_new_handlerv(); +} +#else +std_new_handler_t mi_get_new_handler() { + return NULL; +} +#endif + +static bool mi_try_new_handler(bool nothrow) { + std_new_handler_t h = mi_get_new_handler(); + if (h==NULL) { + if (!nothrow) exit(ENOMEM); + return false; + } + else { + h(); + return true; + } +} +#endif + +static mi_decl_noinline void* mi_try_new(size_t n, bool nothrow ) { + void* p = NULL; + while(p == NULL && mi_try_new_handler(nothrow)) { p = mi_malloc(n); - } while (p==NULL); + } return p; } -// spit out `new_try` for better assembly code -void* mi_new(std::size_t n) noexcept(false) { +void* mi_new(size_t n) { void* p = mi_malloc(n); - if (mi_likely(p != NULL)) return p; - else return mi_new_try(n); -} - -#if (__cplusplus > 201402L || defined(__cpp_aligned_new)) -// for aligned allocation its fine as it is not inlined anyways -void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false) { - void* p; - while ((p = mi_malloc_aligned(n,static_cast(alignment))) == NULL) { - std::new_handler h = std::get_new_handler(); - if (h==NULL) throw std::bad_alloc(); - h(); - // and try again - }; + if (mi_unlikely(p == NULL)) return mi_try_new(n,false); return p; } -#endif -#endif +void* mi_new_aligned(size_t n, size_t alignment) { + void* p; + do { p = mi_malloc_aligned(n, alignment); } + while(p == NULL && mi_try_new_handler(false)); + return p; +} + +void* mi_new_nothrow(size_t n) { + void* p = mi_malloc(n); + if (mi_unlikely(p == NULL)) return mi_try_new(n,true); + return p; +} + +void* mi_new_aligned_nothrow(size_t n, size_t alignment) { + void* p; + do { p = mi_malloc_aligned(n, alignment); } + while (p == NULL && mi_try_new_handler(true)); + return p; +} diff --git a/test/main-override.cpp b/test/main-override.cpp index ee3d22ab..3f2bc960 100644 --- a/test/main-override.cpp +++ b/test/main-override.cpp @@ -5,6 +5,8 @@ #include +#include + static void* p = malloc(8); void free_p() { @@ -36,6 +38,8 @@ int main() { free(s); Test* t = new Test(42); delete t; + t = new (std::nothrow) Test(42); + delete t; int err = mi_posix_memalign(&p1,32,60); if (!err) free(p1); free(p); From c4426e1555fd8d1ec05d2ade9a2c30bda30b3928 Mon Sep 17 00:00:00 2001 From: daan Date: Sun, 14 Jul 2019 20:28:33 -0700 Subject: [PATCH 2/2] add sized delete to windows override --- src/alloc-override-win.c | 54 ++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/src/alloc-override-win.c b/src/alloc-override-win.c index 3ac499b8..f0a5959a 100644 --- a/src/alloc-override-win.c +++ b/src/alloc-override-win.c @@ -98,14 +98,18 @@ static int __cdecl mi_setmaxstdio(int newmax); // Microsoft allocation extensions // ------------------------------------------------------ -#define UNUSED(x) (void)(x) // suppress unused variable warnings - static void* mi__expand(void* p, size_t newsize) { void* res = mi_expand(p, newsize); if (res == NULL) errno = ENOMEM; return res; } +typedef size_t mi_nothrow_t; + +static void mi_free_nothrow(void* p, mi_nothrow_t tag) { + UNUSED(tag); + mi_free(p); +} // Versions of `free`, `realloc`, `recalloc`, `expand` and `msize` // that are used during termination and are no-ops. @@ -113,6 +117,16 @@ static void mi_free_term(void* p) { UNUSED(p); } +static void mi_free_size_term(void* p, size_t size) { + UNUSED(size); + UNUSED(p); +} + +static void mi_free_nothrow_term(void* p, mi_nothrow_t tag) { + UNUSED(tag); + UNUSED(p); +} + static void* mi_realloc_term(void* p, size_t newsize) { UNUSED(p); UNUSED(newsize); return NULL; @@ -444,24 +458,32 @@ static mi_patch_t patches[] = { // override new/delete variants for efficiency (?) #ifdef _WIN64 // 64 bit new/delete - MI_PATCH_NAME2("??2@YAPEAX_K@Z", mi_malloc), - MI_PATCH_NAME2("??_U@YAPEAX_K@Z", mi_malloc), - MI_PATCH_NAME3("??3@YAXPEAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPEAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME2("??2@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_malloc), - MI_PATCH_NAME2("??_U@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_malloc), - MI_PATCH_NAME3("??3@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free, mi_free_term), + MI_PATCH_NAME2("??2@YAPEAX_K@Z", mi_new), + MI_PATCH_NAME2("??_U@YAPEAX_K@Z", mi_new), + MI_PATCH_NAME3("??3@YAXPEAX@Z", mi_free, mi_free_term), + MI_PATCH_NAME3("??_V@YAXPEAX@Z", mi_free, mi_free_term), + MI_PATCH_NAME3("??3@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized + MI_PATCH_NAME3("??_V@YAXPEAX_K@Z", mi_free_size, mi_free_size_term), // delete sized + MI_PATCH_NAME2("??2@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new), + MI_PATCH_NAME2("??_U@YAPEAX_KAEBUnothrow_t@std@@@Z", mi_new), + MI_PATCH_NAME3("??3@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), + MI_PATCH_NAME3("??_V@YAXPEAXAEBUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), + + #else // 32 bit new/delete - MI_PATCH_NAME2("??2@YAPAXI@Z", mi_malloc), - MI_PATCH_NAME2("??_U@YAPAXI@Z", mi_malloc), + MI_PATCH_NAME2("??2@YAPAXI@Z", mi_new), + MI_PATCH_NAME2("??_U@YAPAXI@Z", mi_new), MI_PATCH_NAME3("??3@YAXPAX@Z", mi_free, mi_free_term), MI_PATCH_NAME3("??_V@YAXPAX@Z", mi_free, mi_free_term), - MI_PATCH_NAME2("??2@YAPAXIABUnothrow_t@std@@@Z", mi_malloc), - MI_PATCH_NAME2("??_U@YAPAXIABUnothrow_t@std@@@Z", mi_malloc), - MI_PATCH_NAME3("??3@YAXPAXABUnothrow_t@std@@@Z", mi_free, mi_free_term), - MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free, mi_free_term), + MI_PATCH_NAME3("??3@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized + MI_PATCH_NAME3("??_V@YAXPAXI@Z", mi_free_size, mi_free_size_term), // delete sized + + MI_PATCH_NAME2("??2@YAPAXIABUnothrow_t@std@@@Z", mi_new), + MI_PATCH_NAME2("??_U@YAPAXIABUnothrow_t@std@@@Z", mi_new), + MI_PATCH_NAME3("??3@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), + MI_PATCH_NAME3("??_V@YAXPAXABUnothrow_t@std@@@Z", mi_free_nothrow, mi_free_nothrow_term), + #endif #endif { NULL, NULL, NULL, PATCH_NONE, {NULL,NULL,NULL,NULL} }