From 429025634e7a20983bc5c3b6b946f335b1c5090d Mon Sep 17 00:00:00 2001 From: Haneef Mubarak Date: Tue, 26 May 2020 16:04:28 -0700 Subject: [PATCH 1/7] resolve #201 with a platform-selective REP MOVSB implementation --- include/mimalloc-internal.h | 16 ++++++++++++++++ src/alloc-aligned.c | 2 +- src/alloc-posix.c | 2 +- src/alloc.c | 6 +++--- src/heap.c | 4 ++-- src/init.c | 2 +- src/options.c | 2 +- src/random.c | 2 +- 8 files changed, 26 insertions(+), 10 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 35413315..b191d03d 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -174,6 +174,22 @@ bool _mi_page_is_valid(mi_page_t* page); #define EOVERFLOW (75) #endif +// ------------------------------------------------------ +// Fast `memcpy()` on x86(_64) platforms unavailable, +// use REP MOVSB +// ------------------------------------------------------ + +#if defined(_M_IX86) || defined(_M_X64) +#include +#define _mi_memcpy _mi_memcpy_rep_movsb +static inline void _mi_memcpy_rep_movsb (void *d, const void *s, size_t n) { + __movsb(d, s, n); + return; +} +#else +#define _mi_memcpy memcpy +#endif + /* ----------------------------------------------------------- Inlined definitions diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index ca16d367..10f40355 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -137,7 +137,7 @@ static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t ne memset((uint8_t*)newp + start, 0, newsize - start); } } - memcpy(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy(newp, p, (newsize > size ? size : newsize)); mi_free(p); // only free if successful } return newp; diff --git a/src/alloc-posix.c b/src/alloc-posix.c index 4395893b..de031928 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -99,7 +99,7 @@ mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noex size_t size = (len+1)*sizeof(unsigned short); unsigned short* p = (unsigned short*)mi_malloc(size); if (p != NULL) { - memcpy(p,s,size); + _mi_memcpy(p,s,size); } return p; } diff --git a/src/alloc.c b/src/alloc.c index 607d15b6..33cd2341 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -571,7 +571,7 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); memset((uint8_t*)newp + start, 0, newsize - start); } - memcpy(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy(newp, p, (newsize > size ? size : newsize)); mi_free(p); // only free if successful } return newp; @@ -638,7 +638,7 @@ mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_no if (s == NULL) return NULL; size_t n = strlen(s); char* t = (char*)mi_heap_malloc(heap,n+1); - if (t != NULL) memcpy(t, s, n + 1); + if (t != NULL) _mi_memcpy(t, s, n + 1); return t; } @@ -654,7 +654,7 @@ mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_assert_internal(m <= n); char* t = (char*)mi_heap_malloc(heap, m+1); if (t == NULL) return NULL; - memcpy(t, s, m); + _mi_memcpy(t, s, m); t[m] = 0; return t; } diff --git a/src/heap.c b/src/heap.c index 5d0d4b8a..e9518196 100644 --- a/src/heap.c +++ b/src/heap.c @@ -193,7 +193,7 @@ mi_heap_t* mi_heap_new(void) { mi_heap_t* bheap = mi_heap_get_backing(); mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? if (heap==NULL) return NULL; - memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); + _mi_memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); heap->tld = bheap->tld; heap->thread_id = _mi_thread_id(); _mi_random_split(&bheap->random, &heap->random); @@ -219,7 +219,7 @@ static void mi_heap_reset_pages(mi_heap_t* heap) { #ifdef MI_MEDIUM_DIRECT memset(&heap->pages_free_medium, 0, sizeof(heap->pages_free_medium)); #endif - memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); + _mi_memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); heap->thread_delayed_free = NULL; heap->page_count = 0; } diff --git a/src/init.c b/src/init.c index 132043e8..35a6d4de 100644 --- a/src/init.c +++ b/src/init.c @@ -188,7 +188,7 @@ static bool _mi_heap_init(void) { // OS allocated so already zero initialized mi_tld_t* tld = &td->tld; mi_heap_t* heap = &td->heap; - memcpy(heap, &_mi_heap_empty, sizeof(*heap)); + _mi_memcpy(heap, &_mi_heap_empty, sizeof(*heap)); heap->thread_id = _mi_thread_id(); _mi_random_init(&heap->random); heap->cookie = _mi_heap_random_next(heap) | 1; diff --git a/src/options.c b/src/options.c index f29b387c..7bbb5a70 100644 --- a/src/options.c +++ b/src/options.c @@ -183,7 +183,7 @@ static void mi_out_buf(const char* msg, void* arg) { if (start+n >= MI_MAX_DELAY_OUTPUT) { n = MI_MAX_DELAY_OUTPUT-start-1; } - memcpy(&out_buf[start], msg, n); + _mi_memcpy(&out_buf[start], msg, n); } static void mi_out_buf_flush(mi_output_fun* out, bool no_more_buf, void* arg) { diff --git a/src/random.c b/src/random.c index b3dbf4f8..684559c8 100644 --- a/src/random.c +++ b/src/random.c @@ -115,7 +115,7 @@ static void chacha_init(mi_random_ctx_t* ctx, const uint8_t key[32], uint64_t no static void chacha_split(mi_random_ctx_t* ctx, uint64_t nonce, mi_random_ctx_t* ctx_new) { memset(ctx_new, 0, sizeof(*ctx_new)); - memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); + _mi_memcpy(ctx_new->input, ctx->input, sizeof(ctx_new->input)); ctx_new->input[12] = 0; ctx_new->input[13] = 0; ctx_new->input[14] = (uint32_t)nonce; From 6c92690914094ca6e784ff8d0cd8ee1cfc87d5ed Mon Sep 17 00:00:00 2001 From: Haneef Mubarak Date: Tue, 26 May 2020 16:08:33 -0700 Subject: [PATCH 2/7] fix REP MOVSB doc comment typo --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index b191d03d..43f72a11 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -175,8 +175,8 @@ bool _mi_page_is_valid(mi_page_t* page); #endif // ------------------------------------------------------ -// Fast `memcpy()` on x86(_64) platforms unavailable, -// use REP MOVSB +// Fast `memcpy()` on x86(_64) platforms unavailable +// on Windows, use REP MOVSB if necessary // ------------------------------------------------------ #if defined(_M_IX86) || defined(_M_X64) From 4c45793ec141378f916faef0052356034bc7d678 Mon Sep 17 00:00:00 2001 From: Haneef Mubarak Date: Tue, 26 May 2020 16:16:19 -0700 Subject: [PATCH 3/7] fix __movsb typecast error MSVC --- include/mimalloc-internal.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 43f72a11..6cdf0c03 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -183,7 +183,10 @@ bool _mi_page_is_valid(mi_page_t* page); #include #define _mi_memcpy _mi_memcpy_rep_movsb static inline void _mi_memcpy_rep_movsb (void *d, const void *s, size_t n) { - __movsb(d, s, n); + unsigned char* Destination = (unsigned char*) d; + unsigned const char* Source = (unsigned const char*) s; + size_t Count = n; + __movsb(Destination, Source, Count); return; } #else From 1f08317f3ca30de6df73d6bd6ef83e22c9359de4 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Sat, 30 May 2020 07:19:57 +0200 Subject: [PATCH 4/7] Add option to install directly in CMAKE_INSTALL_PREFIX --- CMakeLists.txt | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e37efcb0..707cf9b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,7 @@ option(MI_BUILD_STATIC "Build static library" ON) option(MI_BUILD_OBJECT "Build object library" ON) option(MI_BUILD_TESTS "Build test executables" ON) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) +option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) include("cmake/mimalloc-config-version.cmake") @@ -175,7 +176,11 @@ endif() # Install and output names # ----------------------------------------------------------------------------- -set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") +if (MI_INSTALL_TOPLEVEL MATCHES "ON") + set(mi_install_dir "${CMAKE_INSTALL_PREFIX}") +else() + set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") +endif() if(MI_SECURE MATCHES "ON") set(mi_basename "mimalloc-secure") else() @@ -227,7 +232,14 @@ if(MI_BUILD_SHARED) COMMENT "Copy mimalloc-redirect.dll to output directory") endif() - install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_dir} LIBRARY) + if (MI_INSTALL_TOPLEVEL MATCHES "ON") + install(TARGETS mimalloc EXPORT mimalloc LIBRARY + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) + else() + install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_dir} LIBRARY) + endif() install(EXPORT mimalloc DESTINATION ${mi_install_dir}/cmake) endif() @@ -261,7 +273,7 @@ install(FILES include/mimalloc-new-delete.h DESTINATION ${mi_install_dir}/includ install(FILES cmake/mimalloc-config.cmake DESTINATION ${mi_install_dir}/cmake) install(FILES cmake/mimalloc-config-version.cmake DESTINATION ${mi_install_dir}/cmake) -if(NOT WIN32 AND MI_BUILD_SHARED) +if(NOT WIN32 AND MI_BUILD_SHARED AND NOT MI_INSTALL_TOPLEVEL) # install a symlink in the /usr/local/lib to the versioned library set(mi_symlink "${CMAKE_SHARED_MODULE_PREFIX}${mi_basename}${CMAKE_SHARED_LIBRARY_SUFFIX}") set(mi_soname "mimalloc-${mi_version}/${mi_symlink}.${mi_version}") From 5291487dac2058c9006e2a9bae0f1e1553e1a991 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 29 Jan 2021 15:52:18 -0800 Subject: [PATCH 5/7] fix cmake typo in merge for #255 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bbc293a8..92dbbad2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,7 +221,7 @@ else() set(mi_install_dir "${CMAKE_INSTALL_PREFIX}/lib/mimalloc-${mi_version}") endif() -if(MI_SECURE MATCHES) +if(MI_SECURE) set(mi_basename "mimalloc-secure") else() set(mi_basename "mimalloc") From 0a06884732b3be074e3e2e18b096c20142ae106c Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 29 Jan 2021 16:09:09 -0800 Subject: [PATCH 6/7] ensure memcpy with rep stosb is only used on windows --- include/mimalloc-internal.h | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index b6bf6f13..da123bf6 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -172,23 +172,19 @@ bool _mi_page_is_valid(mi_page_t* page); #define EOVERFLOW (75) #endif -// ------------------------------------------------------ -// Fast `memcpy()` on x86(_64) platforms unavailable -// on Windows, use REP MOVSB if necessary -// ------------------------------------------------------ -#if defined(_M_IX86) || defined(_M_X64) +// ----------------------------------------------------------------------------------- +// On windows x86/x64 with msvc/clang-cl, use `rep movsb` for `memcpy` (issue #201) +// ----------------------------------------------------------------------------------- + +#if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) #include -#define _mi_memcpy _mi_memcpy_rep_movsb -static inline void _mi_memcpy_rep_movsb (void *d, const void *s, size_t n) { - unsigned char* Destination = (unsigned char*) d; - unsigned const char* Source = (unsigned const char*) s; - size_t Count = n; - __movsb(Destination, Source, Count); - return; +static inline void _mi_memcpy_rep_movsb(void* d, const void* s, size_t n) { + __movsb((unsigned char*)d, (const unsigned char*)s, n); } +#define _mi_memcpy(d,s,n) _mi_memcpy_rep_movsb(d,s,n) #else -#define _mi_memcpy memcpy +#define _mi_memcpy(d,s,n) memcpy(d,s,n) #endif From 92ec493a5d8108cf864f183805d1720242f0e863 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Fri, 29 Jan 2021 16:21:50 -0800 Subject: [PATCH 7/7] possible fix for aligment warning (issue #341) --- include/mimalloc-internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index da123bf6..2ca9d1b8 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -50,8 +50,8 @@ uintptr_t _os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c -extern mi_stats_t _mi_stats_main; -extern const mi_page_t _mi_page_empty; +extern mi_decl_cache_align mi_stats_t _mi_stats_main; +extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); bool _mi_preloading(); // true while the C runtime is not ready