diff --git a/ide/vs2019/mimalloc-override-test.vcxproj b/ide/vs2019/mimalloc-override-test.vcxproj
index a2497a19..7a9202f1 100644
--- a/ide/vs2019/mimalloc-override-test.vcxproj
+++ b/ide/vs2019/mimalloc-override-test.vcxproj
@@ -176,14 +176,14 @@
+
+
+
{abb5eae7-b3e6-432e-b636-333449892ea7}
-
-
-
diff --git a/ide/vs2019/mimalloc-override.vcxproj b/ide/vs2019/mimalloc-override.vcxproj
index 17b6f4c0..a0e79fb0 100644
--- a/ide/vs2019/mimalloc-override.vcxproj
+++ b/ide/vs2019/mimalloc-override.vcxproj
@@ -254,4 +254,4 @@
-
+
\ No newline at end of file
diff --git a/include/mimalloc-internal-tld.h b/include/mimalloc-internal-tld.h
deleted file mode 100644
index ce67b0c7..00000000
--- a/include/mimalloc-internal-tld.h
+++ /dev/null
@@ -1,722 +0,0 @@
-/* ----------------------------------------------------------------------------
-Copyright (c) 2019, Microsoft Research, Daan Leijen
-This is free software; you can redistribute it and/or modify it under the
-terms of the MIT license. A copy of the license can be found in the file
-"LICENSE" at the root of this distribution.
------------------------------------------------------------------------------*/
-#pragma once
-#ifndef MIMALLOC_INTERNAL_TLD_H
-#define MIMALLOC_INTERNAL_TLD_H
-
-#include "mimalloc-types.h"
-#include "mimalloc-internal.h"
-
-#define MI_TLD_DECL 1 // thread local declaration
-#define MI_TLD_PTHREAD 2 // ptrhead_get/setspecific
-#define MI_TLD_DECL_GUARD 3 // thread local + recursion guard at initial load
-#define MI_TLD_PTHREAD_GUARD 4 // ptrhead_get/setspecific + recursion guard at initial load
-#define MI_TLD_SLOT 5 // steal slot from OS thread local predefined slots
-#define MI_TLD_PTHREAD_SLOT 6 // steal slot from pthread structure (usually `retval`)
-
-
-#if !defined(MI_TLD)
-#if defined(_MSC_VER) || defined(__linux__) || defined(__FreeBSD__) || defined(__NetBSD__)
- // on windows and linux/freeBSD/netBSD (with initial-exec) a __thread always works without recursion into malloc
- #define MI_TLD MI_TLD_DECL
-#elif !defined(MI_MIMALLOC_OVERRIDE)
- // if not overriding, __thread declarations should be fine (use MI_TLD_PTHREAD if your OS does not have __thread)
- #define MI_TLD MI_TLD_DECL
-#elif // defined(MI_MALLOC_OVERRIDE)
- // if overriding, some BSD variants allocate when accessing a thread local the first time
- #if defined(__APPLE__)
- #define MI_TLD MI_TLD_SLOT
- #define MI_TLD_SLOT_NUM 89 // seems unused? (__PTK_FRAMEWORK_OLDGC_KEY9) see
- // possibly unused ones are 9, 29, __PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4 (94), __PTK_FRAMEWORK_GC_KEY9 (112) and __PTK_FRAMEWORK_OLDGC_KEY9 (89)
- // #define MI_TLD MI_TLD_PTHREAD_SLOT
- // #define MI_TLD_PTHREAD_SLOT_OFS (2*sizeof(void*) + sizeof(long) + 2*sizeof(void*) /*TAILQ*/) // offset `tl_exit_value`
- #elif defined(__OpenBSD__)
- #define MI_TLD MI_TLD_PTHREAD_SLOT
- #define MI_TLD_PTHREAD_SLOT_OFS (6*sizeof(int) + 1*sizeof(void*)) // offset `retval`
- #elif defined(__DragonFly__)
- #define MI_TLD MI_TLD_PTHREAD_SLOT
- #define MI_TLD_PTHREAD_SLOT_OFS (4 + 1*sizeof(void*)) // offset `uniqueid` (also used by gdb?)
- #endif
- #endif
-#endif
-
-#if (MI_DEBUG>0)
-#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)
-#else
-#define mi_trace_message(...)
-#endif
-
-#define MI_CACHE_LINE 64
-#if defined(_MSC_VER)
-#pragma warning(disable:4127) // suppress constant conditional warning (due to MI_SECURE paths)
-#define mi_decl_noinline __declspec(noinline)
-#define mi_decl_thread __declspec(thread)
-#define mi_decl_cache_align __declspec(align(MI_CACHE_LINE))
-#elif (defined(__GNUC__) && (__GNUC__>=3)) // includes clang and icc
-#define mi_decl_noinline __attribute__((noinline))
-#define mi_decl_thread __thread
-#define mi_decl_cache_align __attribute__((aligned(MI_CACHE_LINE)))
-#else
-#define mi_decl_noinline
-#define mi_decl_thread __thread // hope for the best :-)
-#define mi_decl_cache_align
-#endif
-
-
-// "options.c"
-void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message);
-void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...);
-void _mi_warning_message(const char* fmt, ...);
-void _mi_verbose_message(const char* fmt, ...);
-void _mi_trace_message(const char* fmt, ...);
-void _mi_options_init(void);
-void _mi_error_message(int err, const char* fmt, ...);
-
-// random.c
-void _mi_random_init(mi_random_ctx_t* ctx);
-void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx);
-uintptr_t _mi_random_next(mi_random_ctx_t* ctx);
-uintptr_t _mi_heap_random_next(mi_heap_t* heap);
-uintptr_t _os_random_weak(uintptr_t extra_seed);
-static inline uintptr_t _mi_random_shuffle(uintptr_t x);
-
-// init.c
-extern mi_stats_t _mi_stats_main;
-extern const mi_page_t _mi_page_empty;
-bool _mi_is_main_thread(void);
-bool _mi_preloading(); // true while the C runtime is not ready
-
-// os.c
-size_t _mi_os_page_size(void);
-void _mi_os_init(void); // called from process init
-void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data
-void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data
-size_t _mi_os_good_alloc_size(size_t size);
-
-// memory.c
-void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool* commit, bool* large, bool* is_zero, size_t* id, mi_os_tld_t* tld);
-void _mi_mem_free(void* p, size_t size, size_t id, bool fully_committed, bool any_reset, mi_os_tld_t* tld);
-
-bool _mi_mem_reset(void* p, size_t size, mi_os_tld_t* tld);
-bool _mi_mem_unreset(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld);
-bool _mi_mem_commit(void* p, size_t size, bool* is_zero, mi_os_tld_t* tld);
-bool _mi_mem_protect(void* addr, size_t size);
-bool _mi_mem_unprotect(void* addr, size_t size);
-
-void _mi_mem_collect(mi_os_tld_t* tld);
-
-// "segment.c"
-mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_wsize, mi_segments_tld_t* tld, mi_os_tld_t* os_tld);
-void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld);
-void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld);
-uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size, size_t* pre_size); // page start for any page
-void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
-
-void _mi_segment_thread_collect(mi_segments_tld_t* tld);
-void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
-void _mi_abandoned_await_readers(void);
-
-
-
-// "page.c"
-void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc;
-
-void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks
-void _mi_page_unfull(mi_page_t* page);
-void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page
-void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread...
-void _mi_heap_delayed_free(mi_heap_t* heap);
-void _mi_heap_collect_retired(mi_heap_t* heap, bool force);
-
-void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never);
-size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append);
-void _mi_deferred_free(mi_heap_t* heap, bool force);
-
-void _mi_page_free_collect(mi_page_t* page,bool force);
-void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments
-
-size_t _mi_bin_size(uint8_t bin); // for stats
-uint8_t _mi_bin(size_t size); // for stats
-uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD in "os.c"
-
-// "heap.c"
-void _mi_heap_destroy_pages(mi_heap_t* heap);
-void _mi_heap_collect_abandon(mi_heap_t* heap);
-void _mi_heap_set_default_direct(mi_heap_t* heap);
-
-// "stats.c"
-void _mi_stats_done(mi_stats_t* stats);
-
-mi_msecs_t _mi_clock_now(void);
-mi_msecs_t _mi_clock_end(mi_msecs_t start);
-mi_msecs_t _mi_clock_start(void);
-
-// "alloc.c"
-void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic`
-void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero);
-void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero);
-mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p);
-bool _mi_free_delayed_block(mi_block_t* block);
-void _mi_block_zero_init(const mi_page_t* page, void* p, size_t size);
-
-#if MI_DEBUG>1
-bool _mi_page_is_valid(mi_page_t* page);
-#endif
-
-
-// ------------------------------------------------------
-// Branches
-// ------------------------------------------------------
-
-#if defined(__GNUC__) || defined(__clang__)
-#define mi_unlikely(x) __builtin_expect((x),0)
-#define mi_likely(x) __builtin_expect((x),1)
-#else
-#define mi_unlikely(x) (x)
-#define mi_likely(x) (x)
-#endif
-
-#ifndef __has_builtin
-#define __has_builtin(x) 0
-#endif
-
-
-/* -----------------------------------------------------------
- Error codes passed to `_mi_fatal_error`
- All are recoverable but EFAULT is a serious error and aborts by default in secure mode.
- For portability define undefined error codes using common Unix codes:
-
------------------------------------------------------------ */
-#include
-#ifndef EAGAIN // double free
-#define EAGAIN (11)
-#endif
-#ifndef ENOMEM // out of memory
-#define ENOMEM (12)
-#endif
-#ifndef EFAULT // corrupted free-list or meta-data
-#define EFAULT (14)
-#endif
-#ifndef EINVAL // trying to free an invalid pointer
-#define EINVAL (22)
-#endif
-#ifndef EOVERFLOW // count*size overflow
-#define EOVERFLOW (75)
-#endif
-
-
-/* -----------------------------------------------------------
- Inlined definitions
------------------------------------------------------------ */
-#define UNUSED(x) (void)(x)
-#if (MI_DEBUG>0)
-#define UNUSED_RELEASE(x)
-#else
-#define UNUSED_RELEASE(x) UNUSED(x)
-#endif
-
-#define MI_INIT4(x) x(),x(),x(),x()
-#define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x)
-#define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x)
-#define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x)
-#define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x)
-#define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x)
-#define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x)
-
-
-// Is `x` a power of two? (0 is considered a power of two)
-static inline bool _mi_is_power_of_two(uintptr_t x) {
- return ((x & (x - 1)) == 0);
-}
-
-// Align upwards
-static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
- mi_assert_internal(alignment != 0);
- uintptr_t mask = alignment - 1;
- if ((alignment & mask) == 0) { // power of two?
- return ((sz + mask) & ~mask);
- }
- else {
- return (((sz + mask)/alignment)*alignment);
- }
-}
-
-// Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`.
-static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) {
- mi_assert_internal(divider != 0);
- return (divider == 0 ? size : ((size + divider - 1) / divider));
-}
-
-// Is memory zero initialized?
-static inline bool mi_mem_is_zero(void* p, size_t size) {
- for (size_t i = 0; i < size; i++) {
- if (((uint8_t*)p)[i] != 0) return false;
- }
- return true;
-}
-
-// Align a byte size to a size in _machine words_,
-// i.e. byte size == `wsize*sizeof(void*)`.
-static inline size_t _mi_wsize_from_size(size_t size) {
- mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t));
- return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t);
-}
-
-
-// Overflow detecting multiply
-static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
-#if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5
-#include // UINT_MAX, ULONG_MAX
-#if (SIZE_MAX == UINT_MAX)
- return __builtin_umul_overflow(count, size, total);
-#elif (SIZE_MAX == ULONG_MAX)
- return __builtin_umull_overflow(count, size, total);
-#else
- return __builtin_umulll_overflow(count, size, total);
-#endif
-#else /* __builtin_umul_overflow is unavailable */
- #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
- *total = count * size;
- return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW)
- && size > 0 && (SIZE_MAX / size) < count);
-#endif
-}
-
-// Safe multiply `count*size` into `total`; return `true` on overflow.
-static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) {
- if (count==1) { // quick check for the case where count is one (common for C++ allocators)
- *total = size;
- return false;
- }
- else if (mi_unlikely(mi_mul_overflow(count, size, total))) {
- _mi_error_message(EOVERFLOW, "allocation request too large (%zu * %zu bytes)\n", count, size);
- *total = SIZE_MAX;
- return true;
- }
- else return false;
-}
-
-
-/* -----------------------------------------------------------
- The thread local default heap
------------------------------------------------------------ */
-
-extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap
-extern mi_heap_t _mi_heap_main; // statically allocated main backing heap
-extern bool _mi_process_is_initialized;
-
-#if defined(MI_TLS_OSX_FAST)
-#define MI_TLS_OSX_OFFSET (MI_TLS_OSX_SLOT*sizeof(void*))
-static inline void* mi_tls_osx_fast_get(void) {
- void* ret;
- __asm__("mov %%gs:%1, %0" : "=r" (ret) : "m" (*(void**)(MI_TLS_OSX_OFFSET)));
- return ret;
-}
-static inline void mi_tls_osx_fast_set(void* value) {
- __asm__("movq %1,%%gs:%0" : "=m" (*(void**)(MI_TLS_OSX_OFFSET)) : "rn" (value));
-}
-#elif defined(MI_TLS_PTHREADS)
-extern pthread_key_t _mi_heap_default_key;
-#else
-extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
-#endif
-
-
-static inline mi_heap_t* mi_get_default_heap(void) {
-#if defined(MI_TLS_OSX_FAST)
- // Use a fixed slot in the TSD on MacOSX to avoid recursion (since the loader calls malloc).
- // We use slot 94 (__PTK_FRAMEWORK_JAVASCRIPTCORE_KEY4)
- // which seems unused except for the more recent Webkit
- // Use with care.
- mi_heap_t* heap = (mi_heap_t*)mi_tls_osx_fast_get();
- return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
-#elif defined(MI_TLS_PTHREADS)
- // Use pthreads for TLS; this is used on macOSX with interpose as the loader calls `malloc`
- // to allocate TLS storage leading to recursive calls if __thread declared variables are accessed.
- // Using pthreads allows us to initialize without recursive calls. (performance seems still quite good).
- mi_heap_t* heap = (mi_unlikely(_mi_heap_default_key == (pthread_key_t)(-1)) ? (mi_heap_t*)&_mi_heap_empty : (mi_heap_t*)pthread_getspecific(_mi_heap_default_key));
- return (mi_unlikely(heap == NULL) ? (mi_heap_t*)&_mi_heap_empty : heap);
-#else
- #if defined(MI_TLS_RECURSE_GUARD)
- // On some BSD platforms, like openBSD, the dynamic loader calls `malloc`
- // to initialize thread local data (before our module is loaded).
- // To avoid recursion, we need to avoid accessing the thread local `_mi_default_heap`
- // until our module is loaded and use the statically allocated main heap until that time.
- // TODO: patch ourselves dynamically to avoid this check every time?
- // if (mi_unlikely(!_mi_process_is_initialized)) return &_mi_heap_main;
- #endif
- return _mi_heap_default;
-#endif
-}
-
-static inline bool mi_heap_is_default(const mi_heap_t* heap) {
- return (heap == mi_get_default_heap());
-}
-
-static inline bool mi_heap_is_backing(const mi_heap_t* heap) {
- return (heap->tld->heap_backing == heap);
-}
-
-static inline bool mi_heap_is_initialized(mi_heap_t* heap) {
- mi_assert_internal(heap != NULL);
- return (heap != &_mi_heap_empty);
-}
-
-static inline uintptr_t _mi_ptr_cookie(const void* p) {
- mi_assert_internal(_mi_heap_main.cookie != 0);
- return ((uintptr_t)p ^ _mi_heap_main.cookie);
-}
-
-/* -----------------------------------------------------------
- Pages
------------------------------------------------------------ */
-
-static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) {
- mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE));
- const size_t idx = _mi_wsize_from_size(size);
- mi_assert_internal(idx < MI_PAGES_DIRECT);
- return heap->pages_free_direct[idx];
-}
-
-// Get the page belonging to a certain size class
-static inline mi_page_t* _mi_get_free_small_page(size_t size) {
- return _mi_heap_get_free_small_page(mi_get_default_heap(), size);
-}
-
-// Segment that contains the pointer
-static inline mi_segment_t* _mi_ptr_segment(const void* p) {
- // mi_assert_internal(p != NULL);
- return (mi_segment_t*)((uintptr_t)p & ~MI_SEGMENT_MASK);
-}
-
-// Segment belonging to a page
-static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) {
- mi_segment_t* segment = _mi_ptr_segment(page);
- mi_assert_internal(segment == NULL || page == &segment->pages[page->segment_idx]);
- return segment;
-}
-
-// used internally
-static inline uintptr_t _mi_segment_page_idx_of(const mi_segment_t* segment, const void* p) {
- // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages
- ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment;
- mi_assert_internal(diff >= 0 && (size_t)diff < MI_SEGMENT_SIZE);
- uintptr_t idx = (uintptr_t)diff >> segment->page_shift;
- mi_assert_internal(idx < segment->capacity);
- mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0);
- return idx;
-}
-
-// Get the page containing the pointer
-static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) {
- uintptr_t idx = _mi_segment_page_idx_of(segment, p);
- return &((mi_segment_t*)segment)->pages[idx];
-}
-
-// Quick page start for initialized pages
-static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) {
- const size_t bsize = page->xblock_size;
- mi_assert_internal(bsize > 0 && (bsize%sizeof(void*)) == 0);
- return _mi_segment_page_start(segment, page, bsize, page_size, NULL);
-}
-
-// Get the page containing the pointer
-static inline mi_page_t* _mi_ptr_page(void* p) {
- return _mi_segment_page_of(_mi_ptr_segment(p), p);
-}
-
-// Get the block size of a page (special cased for huge objects)
-static inline size_t mi_page_block_size(const mi_page_t* page) {
- const size_t bsize = page->xblock_size;
- mi_assert_internal(bsize > 0);
- if (mi_likely(bsize < MI_HUGE_BLOCK_SIZE)) {
- return bsize;
- }
- else {
- size_t psize;
- _mi_segment_page_start(_mi_page_segment(page), page, bsize, &psize, NULL);
- return psize;
- }
-}
-
-// Get the usable block size of a page without fixed padding.
-// This may still include internal padding due to alignment and rounding up size classes.
-static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
- return mi_page_block_size(page) - MI_PADDING_SIZE;
-}
-
-
-// Thread free access
-static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) {
- return (mi_block_t*)(mi_atomic_read_relaxed(&page->xthread_free) & ~3);
-}
-
-static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) {
- return (mi_delayed_t)(mi_atomic_read_relaxed(&page->xthread_free) & 3);
-}
-
-// Heap access
-static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
- return (mi_heap_t*)(mi_atomic_read_relaxed(&page->xheap));
-}
-
-static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {
- mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING);
- mi_atomic_write(&page->xheap,(uintptr_t)heap);
-}
-
-// Thread free flag helpers
-static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) {
- return (mi_block_t*)(tf & ~0x03);
-}
-static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) {
- return (mi_delayed_t)(tf & 0x03);
-}
-static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) {
- return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed);
-}
-static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) {
- return mi_tf_make(mi_tf_block(tf),delayed);
-}
-static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) {
- return mi_tf_make(block, mi_tf_delayed(tf));
-}
-
-// are all blocks in a page freed?
-// note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`.
-static inline bool mi_page_all_free(const mi_page_t* page) {
- mi_assert_internal(page != NULL);
- return (page->used == 0);
-}
-
-// are there any available blocks?
-static inline bool mi_page_has_any_available(const mi_page_t* page) {
- mi_assert_internal(page != NULL && page->reserved > 0);
- return (page->used < page->reserved || (mi_page_thread_free(page) != NULL));
-}
-
-// are there immediately available blocks, i.e. blocks available on the free list.
-static inline bool mi_page_immediate_available(const mi_page_t* page) {
- mi_assert_internal(page != NULL);
- return (page->free != NULL);
-}
-
-// is more than 7/8th of a page in use?
-static inline bool mi_page_mostly_used(const mi_page_t* page) {
- if (page==NULL) return true;
- uint16_t frac = page->reserved / 8U;
- return (page->reserved - page->used <= frac);
-}
-
-static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) {
- return &((mi_heap_t*)heap)->pages[_mi_bin(size)];
-}
-
-
-
-//-----------------------------------------------------------
-// Page flags
-//-----------------------------------------------------------
-static inline bool mi_page_is_in_full(const mi_page_t* page) {
- return page->flags.x.in_full;
-}
-
-static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
- page->flags.x.in_full = in_full;
-}
-
-static inline bool mi_page_has_aligned(const mi_page_t* page) {
- return page->flags.x.has_aligned;
-}
-
-static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) {
- page->flags.x.has_aligned = has_aligned;
-}
-
-
-/* -------------------------------------------------------------------
-Encoding/Decoding the free list next pointers
-
-This is to protect against buffer overflow exploits where the
-free list is mutated. Many hardened allocators xor the next pointer `p`
-with a secret key `k1`, as `p^k1`. This prevents overwriting with known
-values but might be still too weak: if the attacker can guess
-the pointer `p` this can reveal `k1` (since `p^k1^p == k1`).
-Moreover, if multiple blocks can be read as well, the attacker can
-xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot
-about the pointers (and subsequently `k1`).
-
-Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<> (MI_INTPTR_BITS - shift)));
-}
-static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) {
- shift %= MI_INTPTR_BITS;
- return ((x >> shift) | (x << (MI_INTPTR_BITS - shift)));
-}
-
-static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) {
- void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]);
- return (mi_unlikely(p==null) ? NULL : p);
-}
-
-static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) {
- uintptr_t x = (uintptr_t)(mi_unlikely(p==NULL) ? null : p);
- return mi_rotl(x ^ keys[1], keys[0]) + keys[0];
-}
-
-static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
- #ifdef MI_ENCODE_FREELIST
- return (mi_block_t*)mi_ptr_decode(null, block->next, keys);
- #else
- UNUSED(keys); UNUSED(null);
- return (mi_block_t*)block->next;
- #endif
-}
-
-static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {
- #ifdef MI_ENCODE_FREELIST
- block->next = mi_ptr_encode(null, next, keys);
- #else
- UNUSED(keys); UNUSED(null);
- block->next = (mi_encoded_t)next;
- #endif
-}
-
-static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {
- #ifdef MI_ENCODE_FREELIST
- mi_block_t* next = mi_block_nextx(page,block,page->keys);
- // check for free list corruption: is `next` at least in the same page?
- // TODO: check if `next` is `page->block_size` aligned?
- if (mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next))) {
- _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next);
- next = NULL;
- }
- return next;
- #else
- UNUSED(page);
- return mi_block_nextx(page,block,NULL);
- #endif
-}
-
-static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {
- #ifdef MI_ENCODE_FREELIST
- mi_block_set_nextx(page,block,next, page->keys);
- #else
- UNUSED(page);
- mi_block_set_nextx(page,block,next,NULL);
- #endif
-}
-
-// -------------------------------------------------------------------
-// Fast "random" shuffle
-// -------------------------------------------------------------------
-
-static inline uintptr_t _mi_random_shuffle(uintptr_t x) {
- if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros
-#if (MI_INTPTR_SIZE==8)
- // by Sebastiano Vigna, see:
- x ^= x >> 30;
- x *= 0xbf58476d1ce4e5b9UL;
- x ^= x >> 27;
- x *= 0x94d049bb133111ebUL;
- x ^= x >> 31;
-#elif (MI_INTPTR_SIZE==4)
- // by Chris Wellons, see:
- x ^= x >> 16;
- x *= 0x7feb352dUL;
- x ^= x >> 15;
- x *= 0x846ca68bUL;
- x ^= x >> 16;
-#endif
- return x;
-}
-
-// -------------------------------------------------------------------
-// Optimize numa node access for the common case (= one node)
-// -------------------------------------------------------------------
-
-int _mi_os_numa_node_get(mi_os_tld_t* tld);
-size_t _mi_os_numa_node_count_get(void);
-
-extern size_t _mi_numa_node_count;
-static inline int _mi_os_numa_node(mi_os_tld_t* tld) {
- if (mi_likely(_mi_numa_node_count == 1)) return 0;
- else return _mi_os_numa_node_get(tld);
-}
-static inline size_t _mi_os_numa_node_count(void) {
- if (mi_likely(_mi_numa_node_count>0)) return _mi_numa_node_count;
- else return _mi_os_numa_node_count_get();
-}
-
-
-// -------------------------------------------------------------------
-// Getting the thread id should be performant
-// as it is called in the fast path of `_mi_free`,
-// so we specialize for various platforms.
-// -------------------------------------------------------------------
-#if defined(_WIN32)
-#define WIN32_LEAN_AND_MEAN
-#include
-static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
- // Windows: works on Intel and ARM in both 32- and 64-bit
- return (uintptr_t)NtCurrentTeb();
-}
-#elif (defined(__GNUC__) || defined(__clang__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))
-// TLS register on x86 is in the FS or GS register
-// see: https://akkadia.org/drepper/tls.pdf
-static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
- uintptr_t tid;
- #if defined(__i386__)
- __asm__("movl %%gs:0, %0" : "=r" (tid) : : ); // 32-bit always uses GS
- #elif defined(__MACH__)
- __asm__("movq %%gs:0, %0" : "=r" (tid) : : ); // x86_64 macOS uses GS
- #elif defined(__x86_64__)
- __asm__("movq %%fs:0, %0" : "=r" (tid) : : ); // x86_64 Linux, BSD uses FS
- #elif defined(__arm__)
- asm volatile ("mrc p15, 0, %0, c13, c0, 3" : "=r" (tid));
- #elif defined(__aarch64__)
- asm volatile ("mrs %0, tpidr_el0" : "=r" (tid));
- #endif
- return tid;
-}
-#else
-// otherwise use standard C
-static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept {
- return (uintptr_t)&_mi_heap_default;
-}
-#endif
-
-
-#endif
diff --git a/include/mimalloc.h b/include/mimalloc.h
index eccd02c8..13c50898 100644
--- a/include/mimalloc.h
+++ b/include/mimalloc.h
@@ -34,7 +34,7 @@ terms of the MIT license. A copy of the license can be found in the file
#define mi_decl_nodiscard
#endif
-#ifdef _MSC_VER
+#if defined(_MSC_VER) || defined(__MINGW32__)
#if !defined(MI_SHARED_LIB)
#define mi_decl_export
#elif defined(MI_SHARED_LIB_EXPORT)
@@ -42,13 +42,18 @@ terms of the MIT license. A copy of the license can be found in the file
#else
#define mi_decl_export __declspec(dllimport)
#endif
- #if (_MSC_VER >= 1900) && !defined(__EDG__)
- #define mi_decl_restrict __declspec(allocator) __declspec(restrict)
+ #if defined(__MINGW32__)
+ #define mi_decl_restrict
+ #define mi_attr_malloc __attribute__((malloc))
#else
- #define mi_decl_restrict __declspec(restrict)
+ #if (_MSC_VER >= 1900) && !defined(__EDG__)
+ #define mi_decl_restrict __declspec(allocator) __declspec(restrict)
+ #else
+ #define mi_decl_restrict __declspec(restrict)
+ #endif
+ #define mi_attr_malloc
#endif
#define mi_cdecl __cdecl
- #define mi_attr_malloc
#define mi_attr_alloc_size(s)
#define mi_attr_alloc_size2(s1,s2)
#define mi_attr_alloc_align(p)
diff --git a/src/alloc.c b/src/alloc.c
index 449ea6fb..0498c767 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -474,15 +474,16 @@ size_t mi_usable_size(const void* p) mi_attr_noexcept {
if (p==NULL) return 0;
const mi_segment_t* const segment = _mi_ptr_segment(p);
const mi_page_t* const page = _mi_segment_page_of(segment, p);
- const mi_block_t* const block = (const mi_block_t*)p;
- const size_t size = mi_page_usable_size_of(page, block);
+ const mi_block_t* block = (const mi_block_t*)p;
if (mi_unlikely(mi_page_has_aligned(page))) {
- ptrdiff_t const adjust = (uint8_t*)p - (uint8_t*)_mi_page_ptr_unalign(segment,page,p);
+ block = _mi_page_ptr_unalign(segment, page, p);
+ size_t size = mi_page_usable_size_of(page, block);
+ ptrdiff_t const adjust = (uint8_t*)p - (uint8_t*)block;
mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
return (size - adjust);
}
else {
- return size;
+ return mi_page_usable_size_of(page, block);
}
}
diff --git a/src/init.c b/src/init.c
index 32470c1c..3bcc7094 100644
--- a/src/init.c
+++ b/src/init.c
@@ -286,6 +286,12 @@ static void _mi_thread_done(mi_heap_t* default_heap);
// use thread local storage keys to detect thread ending
#include
#include
+ #if (_WIN32_WINNT < 0x600) // before Windows Vista
+ WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
+ WINBASEAPI PVOID WINAPI FlsGetValue( _In_ DWORD dwFlsIndex );
+ WINBASEAPI BOOL WINAPI FlsSetValue( _In_ DWORD dwFlsIndex, _In_opt_ PVOID lpFlsData );
+ WINBASEAPI BOOL WINAPI FlsFree(_In_ DWORD dwFlsIndex);
+ #endif
static DWORD mi_fls_key = (DWORD)(-1);
static void NTAPI mi_fls_done(PVOID value) {
if (value!=NULL) _mi_thread_done((mi_heap_t*)value);
diff --git a/src/os.c b/src/os.c
index 89fd349b..12ba9442 100644
--- a/src/os.c
+++ b/src/os.c
@@ -823,7 +823,7 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`)
-----------------------------------------------------------------------------*/
#define MI_HUGE_OS_PAGE_SIZE (GiB)
-#if defined(WIN32) && (MI_INTPTR_SIZE >= 8)
+#if defined(_WIN32) && (MI_INTPTR_SIZE >= 8)
static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
{
mi_assert_internal(size%GiB == 0);
@@ -866,6 +866,8 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
params[0].ULong = (unsigned)numa_node;
return (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, params, 1);
}
+ #else
+ UNUSED(numa_node);
#endif
// otherwise use regular virtual alloc on older windows
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
@@ -905,6 +907,7 @@ static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node)
}
#else
static void* mi_os_alloc_huge_os_pagesx(void* addr, size_t size, int numa_node) {
+ UNUSED(addr); UNUSED(size); UNUSED(numa_node);
return NULL;
}
#endif
@@ -940,6 +943,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
}
#else
static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
+ UNUSED(pages);
if (total_size != NULL) *total_size = 0;
return NULL;
}
@@ -1012,7 +1016,12 @@ void _mi_os_free_huge_pages(void* p, size_t size, mi_stats_t* stats) {
/* ----------------------------------------------------------------------------
Support NUMA aware allocation
-----------------------------------------------------------------------------*/
-#ifdef WIN32
+#ifdef _WIN32
+ #if (_WIN32_WINNT < 0x601) // before Win7
+ typedef struct _PROCESSOR_NUMBER { WORD Group; BYTE Number; BYTE Reserved; } PROCESSOR_NUMBER, *PPROCESSOR_NUMBER;
+ WINBASEAPI VOID WINAPI GetCurrentProcessorNumberEx(_Out_ PPROCESSOR_NUMBER ProcNumber);
+ WINBASEAPI BOOL WINAPI GetNumaProcessorNodeEx(_In_ PPROCESSOR_NUMBER Processor, _Out_ PUSHORT NodeNumber);
+ #endif
static size_t mi_os_numa_nodex() {
PROCESSOR_NUMBER pnum;
USHORT numa_node = 0;
diff --git a/src/segment.c b/src/segment.c
index 9eab7548..8aa838a2 100644
--- a/src/segment.c
+++ b/src/segment.c
@@ -1290,7 +1290,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block
mi_assert_internal(mi_atomic_read_relaxed(&segment->thread_id)==0);
// claim it and free
- mi_heap_t* heap = mi_get_default_heap();
+ mi_heap_t* heap = mi_heap_get_default(); // issue #221; don't use the internal get_default_heap as we need to ensure the thread is initialized.
// paranoia: if this it the last reference, the cas should always succeed
if (mi_atomic_cas_strong(&segment->thread_id, heap->thread_id, 0)) {
mi_block_set_next(page, block, page->free);
@@ -1298,16 +1298,16 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block
page->used--;
page->is_zero = false;
mi_assert(page->used == 0);
- mi_segments_tld_t* tld = &heap->tld->segments;
+ mi_tld_t* tld = heap->tld;
const size_t bsize = mi_page_usable_block_size(page);
if (bsize > MI_HUGE_OBJ_SIZE_MAX) {
- _mi_stat_decrease(&tld->stats->giant, bsize);
+ _mi_stat_decrease(&tld->stats.giant, bsize);
}
else {
- _mi_stat_decrease(&tld->stats->huge, bsize);
+ _mi_stat_decrease(&tld->stats.huge, bsize);
}
- mi_segments_track_size((long)segment->segment_size, tld);
- _mi_segment_page_free(page, true, tld);
+ mi_segments_track_size((long)segment->segment_size, &tld->segments);
+ _mi_segment_page_free(page, true, &tld->segments);
}
}
diff --git a/test/main-override.cpp b/test/main-override.cpp
index 66996959..922c73ab 100644
--- a/test/main-override.cpp
+++ b/test/main-override.cpp
@@ -18,17 +18,18 @@ static void msleep(unsigned long msecs) { Sleep(msecs); }
static void msleep(unsigned long msecs) { usleep(msecs * 1000UL); }
#endif
-static void heap_no_delete();
-static void heap_late_free();
-static void various_tests();
-static void dangling_ptr_write();
+void heap_thread_free_large(); // issue #221
+void heap_no_delete(); // issue #202
+void heap_late_free(); // issue #204
+void padding_shrink(); // issue #209
+void various_tests();
int main() {
mi_stats_reset(); // ignore earlier allocations
- // heap_no_delete(); // issue #202
- // heap_late_free(); // issue #204
- // dangling_ptr_write();
- // padding_shrink(); // issue #209
+ heap_thread_free_large();
+ heap_no_delete();
+ heap_late_free();
+ padding_shrink();
various_tests();
mi_stats_print(NULL);
return 0;
@@ -70,7 +71,7 @@ static void various_tests() {
free(p1);
free(p2);
mi_free(s);
- s[0] = 0;
+ // s[0] = 0; // test corruption
Test* t = new Test(42);
delete t;
// t = new(std::nothrow) Test(42); // does not work with overriding :-(
@@ -175,3 +176,16 @@ void padding_shrink(void)
mi_free(shared_p);
}
+
+// Issue #221
+void heap_thread_free_large_worker() {
+ mi_free(shared_p);
+}
+
+void heap_thread_free_large() {
+ for (int i = 0; i < 100; i++) {
+ shared_p = mi_malloc_aligned(2*1024*1024 + 1, 8);
+ auto t1 = std::thread(heap_thread_free_large_worker);
+ t1.join();
+ }
+}
diff --git a/test/test-api.c b/test/test-api.c
index e274fd96..0b911b82 100644
--- a/test/test-api.c
+++ b/test/test-api.c
@@ -152,6 +152,9 @@ int main() {
}
result = ok;
});
+ CHECK_BODY("malloc-aligned5", {
+ void* p = mi_malloc_aligned(4097,4096); size_t usable = mi_usable_size(p); result = usable >= 4097 && usable < 10000; mi_free(p);
+ });
CHECK_BODY("malloc-aligned-at1", {
void* p = mi_malloc_aligned_at(48,32,0); result = (p != NULL && ((uintptr_t)(p) + 0) % 32 == 0); mi_free(p);
});