3party/mimalloc
0
0
Fork 0
mirror of https://github.com/microsoft/mimalloc.git synced 2024-12-27 13:33:18 +08:00
Code Issues Projects Releases Wiki Activity

wip: increase commit mask resolution

Browse Source
This commit is contained in:
daan 2021-11-09 20:19:31 -08:00
parent c17878d1a7
commit 49c75a3157
6 changed files with 255 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
include/mimalloc-internal.h
View File

@ -86,7 +86,7 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed,
// "segment-cache.c" // "segment-cache.c"
void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld);
bool _mi_segment_cache_push(void* start, size_t size, size_t memid, mi_commit_mask_t commit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld); bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld);
void _mi_segment_map_allocated_at(const mi_segment_t* segment); void _mi_segment_map_allocated_at(const mi_segment_t* segment);
void _mi_segment_map_freed_at(const mi_segment_t* segment); void _mi_segment_map_freed_at(const mi_segment_t* segment);
@ -691,77 +691,163 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c
// commit mask // commit mask
// ------------------------------------------------------------------- // -------------------------------------------------------------------
#define MI_COMMIT_MASK_BITS (sizeof(mi_commit_mask_t)*8)
static inline mi_commit_mask_t mi_commit_mask_empty(void) { static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) {
return 0; memset(cm, 0, sizeof(*cm));
} }
static inline mi_commit_mask_t mi_commit_mask_full(void) { static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) {
return ~mi_commit_mask_empty(); memset(cm, 0xFF, sizeof(*cm));
} }
static inline mi_commit_mask_t mi_commit_mask_create(uintptr_t bitidx, uintptr_t bitcount) { static inline void mi_commit_mask_create(ptrdiff_t bitidx, ptrdiff_t bitcount, mi_commit_mask_t* cm) {
mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS); mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS); mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
if (bitcount == MI_COMMIT_MASK_BITS) { if (bitcount == MI_COMMIT_MASK_BITS) {
mi_assert_internal(bitidx==0); mi_assert_internal(bitidx==0);
return mi_commit_mask_full(); mi_commit_mask_create_full(cm);
} }
else if (bitcount == 0) { else if (bitcount == 0) {
return mi_commit_mask_empty(); mi_commit_mask_create_empty(cm);
} }
else { else {
return (((uintptr_t)1 << bitcount) - 1) << bitidx; mi_commit_mask_create_empty(cm);
ptrdiff_t i = bitidx / MI_COMMIT_MASK_FIELD_BITS;
ptrdiff_t ofs = bitidx % MI_COMMIT_MASK_FIELD_BITS;
while (bitcount > 0) {
mi_assert_internal(i < MI_COMMIT_MASK_N);
ptrdiff_t avail = MI_COMMIT_MASK_FIELD_BITS - ofs;
ptrdiff_t count = (bitcount > avail ? avail : bitcount);
size_t mask = (((size_t)1 << count) - 1) << ofs;
cm->mask[i] = mask;
bitcount -= count;
ofs = 0;
i++;
}
} }
} }
static inline bool mi_commit_mask_is_empty(mi_commit_mask_t mask) { static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) {
return (mask == 0); for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
} if (cm->mask[i] != 0) return false;
static inline bool mi_commit_mask_is_full(mi_commit_mask_t mask) {
return ((~mask) == 0);
}
static inline bool mi_commit_mask_all_set(mi_commit_mask_t commit, mi_commit_mask_t mask) {
return ((commit & mask) == mask);
}
static inline bool mi_commit_mask_any_set(mi_commit_mask_t commit, mi_commit_mask_t mask) {
return ((commit & mask) != 0);
}
mi_decl_nodiscard static inline mi_commit_mask_t mi_commit_mask_intersect(mi_commit_mask_t commit, mi_commit_mask_t mask) {
return (commit & mask);
}
static inline void mi_commit_mask_clear(mi_commit_mask_t* commit, mi_commit_mask_t mask) {
*commit = (*commit) & (~mask);
}
static inline void mi_commit_mask_set(mi_commit_mask_t* commit, mi_commit_mask_t mask) {
*commit = (*commit) | mask;
}
static inline size_t mi_commit_mask_committed_size(mi_commit_mask_t mask, size_t total) {
if (mi_commit_mask_is_full(mask)) {
return total;
} }
else if (mi_commit_mask_is_empty(mask)) { return true;
}
static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
if (cm->mask[i] != 0) return false;
}
return true;
}
static inline bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
if ((commit->mask[i] & cm->mask[i]) != cm->mask[i]) return false;
}
return true;
}
static inline bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
if ((commit->mask[i] & cm->mask[i]) != 0) return true;
}
return false;
}
static inline void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
res->mask[i] = (commit->mask[i] & cm->mask[i]);
}
}
static inline void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
res->mask[i] &= ~(cm->mask[i]);
}
}
static inline void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) {
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
res->mask[i] |= cm->mask[i];
}
}
static inline size_t mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) {
mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
size_t count = 0;
for (ptrdiff_t i = 0; i < MI_COMMIT_MASK_N; i++) {
size_t mask = cm->mask[i];
if (~mask == 0) {
count += MI_COMMIT_MASK_FIELD_BITS;
}
else {
for (; mask != 0; mask >>= 1) { // todo: use popcount
if ((mask&1)!=0) count++;
}
}
}
// we use total since for huge segments each commit bit may represent a larger size
return (total / MI_COMMIT_MASK_BITS)* count;
}
static inline ptrdiff_t mi_commit_mask_next_run(const mi_commit_mask_t* cm, ptrdiff_t* idx ) {
ptrdiff_t i = (*idx) / MI_COMMIT_MASK_FIELD_BITS;
ptrdiff_t ofs = (*idx) % MI_COMMIT_MASK_FIELD_BITS;
size_t mask = 0;
// find first ones
while (i < MI_COMMIT_MASK_N) {
mask = cm->mask[i];
mask >>= ofs;
if (mask != 0) {
while ((mask&1) == 0) {
mask >>= 1;
ofs++;
}
break;
}
i++;
ofs = 0;
}
if (i >= MI_COMMIT_MASK_N) {
// not found
*idx = MI_COMMIT_MASK_BITS;
return 0; return 0;
} }
else { else {
size_t count = 0; // found, count ones
for (; mask != 0; mask >>= 1) { // todo: use popcount ptrdiff_t count = 0;
if ((mask&1)!=0) count++; *idx = (i*MI_COMMIT_MASK_FIELD_BITS) + ofs;
} mi_assert_internal(ofs < MI_COMMIT_MASK_FIELD_BITS && (mask&1) == 1);
return (total/MI_COMMIT_MASK_BITS)*count; do {
do {
count++;
mask >>= 1;
} while (mask != 0);
if ((((count + ofs) % MI_COMMIT_MASK_FIELD_BITS) == 0)) {
i++;
if (i >= MI_COMMIT_MASK_N) break;
mask = cm->mask[i];
if ((mask&1)==0) break;
ofs = 0;
}
} while (mask != 0);
mi_assert_internal(count > 0);
return count;
} }
} }
#define mi_commit_mask_foreach(cm,idx,count) \
idx = 0; \
while ((count = mi_commit_mask_next_run(cm,&idx)) > 0) {
#define mi_commit_mask_foreach_end() \
idx += count; \
}
#define mi_commit_mask_foreach(mask,idx,count) \ #define xmi_commit_mask_foreach(mask,idx,count) \
idx = 0; \ idx = 0; \
while (mask != 0) { \ while (mask != 0) { \
/* count ones */ \ /* count ones */ \
@ -773,7 +859,7 @@ static inline size_t mi_commit_mask_committed_size(mi_commit_mask_t mask, size_t
/* if found, do action */ \ /* if found, do action */ \
if (count > 0) { if (count > 0) {
#define mi_commit_mask_foreach_end() \ #define xmi_commit_mask_foreach_end() \
} \ } \
idx += count; \ idx += count; \
/* shift out the zero */ \ /* shift out the zero */ \

31
include/mimalloc-types.h
View File

@ -83,9 +83,9 @@ terms of the MIT license. A copy of the license can be found in the file
// or otherwise one might define an intptr_t type that is larger than a pointer... // or otherwise one might define an intptr_t type that is larger than a pointer...
// ------------------------------------------------------ // ------------------------------------------------------
#if INTPTR_MAX == 9223372036854775807LL #if INTPTR_MAX == INT64_MAX
# define MI_INTPTR_SHIFT (3) # define MI_INTPTR_SHIFT (3)
#elif INTPTR_MAX == 2147483647LL #elif INTPTR_MAX == INT32_MAX
# define MI_INTPTR_SHIFT (2) # define MI_INTPTR_SHIFT (2)
#else #else
#error platform must be 32 or 64 bits #error platform must be 32 or 64 bits
@ -94,6 +94,18 @@ terms of the MIT license. A copy of the license can be found in the file
#define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT) #define MI_INTPTR_SIZE (1<<MI_INTPTR_SHIFT)
#define MI_INTPTR_BITS (MI_INTPTR_SIZE*8) #define MI_INTPTR_BITS (MI_INTPTR_SIZE*8)
#if SIZE_MAX == UINT64_MAX
# define MI_SIZE_SHIFT (3)
#elif SIZE_MAX == UINT32_MAX
# define MI_SIZE_SHIFT (2)
#else
#error platform must be 32 or 64 bits
#endif
#define MI_SIZE_SIZE (1<<MI_INTPTR_SHIFT)
#define MI_SIZE_BITS (MI_INTPTR_SIZE*8)
#define KiB ((size_t)1024) #define KiB ((size_t)1024)
#define MiB (KiB*KiB) #define MiB (KiB*KiB)
#define GiB (MiB*KiB) #define GiB (MiB*KiB)
@ -275,17 +287,22 @@ typedef enum mi_segment_kind_e {
MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside. MI_SEGMENT_HUGE, // > MI_LARGE_SIZE_MAX segment with just one huge page inside.
} mi_segment_kind_t; } mi_segment_kind_t;
#define MI_COMMIT_SIZE (MI_SEGMENT_SIZE/MI_INTPTR_BITS) #define MI_COMMIT_SIZE (128*1024)
#define MI_COMMIT_MASK_BITS (MI_SEGMENT_SIZE / MI_COMMIT_SIZE)
#define MI_COMMIT_MASK_FIELD_BITS MI_SIZE_BITS
#define MI_COMMIT_MASK_N (MI_COMMIT_MASK_BITS / MI_COMMIT_MASK_FIELD_BITS)
#if (((1 << MI_SEGMENT_SHIFT)/MI_COMMIT_SIZE) > 8*MI_INTPTR_SIZE) #if (MI_COMMIT_MASK_BITS != (MI_COMMIT_MASK_N * MI_COMMIT_MASK_FIELD_BITS))
#error "not enough commit bits to cover the segment size" #error "the segment size must be exactly divisible by the (commit size * size_t bits)"
#endif #endif
typedef mi_page_t mi_slice_t; typedef struct mi_commit_mask_s {
size_t mask[MI_COMMIT_MASK_N];
} mi_commit_mask_t;
typedef mi_page_t mi_slice_t;
typedef int64_t mi_msecs_t; typedef int64_t mi_msecs_t;
typedef uintptr_t mi_commit_mask_t;
// Segments are large allocated memory blocks (8mb on 64 bit) from // Segments are large allocated memory blocks (8mb on 64 bit) from
// the OS. Inside segments we allocated fixed size _pages_ that // the OS. Inside segments we allocated fixed size _pages_ that

4
src/options.c
View File

@ -66,7 +66,7 @@ static mi_option_desc_t options[_mi_option_last] =
{ 0, UNINIT, MI_OPTION(verbose) }, { 0, UNINIT, MI_OPTION(verbose) },
// the following options are experimental and not all combinations make sense. // the following options are experimental and not all combinations make sense.
{ 1, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (8MiB) (but see also `eager_commit_delay`) { 0, UNINIT, MI_OPTION(eager_commit) }, // commit per segment directly (8MiB) (but see also `eager_commit_delay`)
#if defined(_WIN32) || (MI_INTPTR_SIZE <= 4) // and other OS's without overcommit? #if defined(_WIN32) || (MI_INTPTR_SIZE <= 4) // and other OS's without overcommit?
{ 0, UNINIT, MI_OPTION(eager_region_commit) }, { 0, UNINIT, MI_OPTION(eager_region_commit) },
{ 0, UNINIT, MI_OPTION(reset_decommits) }, // reset decommits memory { 0, UNINIT, MI_OPTION(reset_decommits) }, // reset decommits memory
@ -89,7 +89,7 @@ static mi_option_desc_t options[_mi_option_last] =
{ 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) { 1, UNINIT, MI_OPTION(eager_commit_delay) }, // the first N segments per thread are not eagerly committed (but per page in the segment on demand)
#endif #endif
{ 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit slices when no longer used (after reset_delay milli-seconds) { 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit slices when no longer used (after reset_delay milli-seconds)
{ 500, UNINIT, MI_OPTION(reset_delay) }, // page reset delay in milli-seconds (= decommit) { 10, UNINIT, MI_OPTION(reset_delay) }, // page reset delay in milli-seconds (= decommit)
{ 1000, UNINIT, MI_OPTION(segment_decommit_delay) },// decommit delay in milli-seconds for freed segments { 1000, UNINIT, MI_OPTION(segment_decommit_delay) },// decommit delay in milli-seconds for freed segments
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes. { 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
{ 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas) { 0, UNINIT, MI_OPTION(limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)

24
src/segment-cache.c
View File

@ -76,11 +76,10 @@ mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* comm
*memid = slot->memid; *memid = slot->memid;
*is_pinned = slot->is_pinned; *is_pinned = slot->is_pinned;
*is_zero = false; *is_zero = false;
mi_commit_mask_t cmask = slot->commit_mask; // copy *commit_mask = slot->commit_mask;
slot->p = NULL; slot->p = NULL;
mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_release(&slot->expire,(mi_msecs_t)0);
*commit_mask = cmask;
// mark the slot as free again // mark the slot as free again
mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(_mi_bitmap_is_claimed(cache_inuse, MI_CACHE_FIELDS, 1, bitidx));
_mi_bitmap_unclaim(cache_inuse, MI_CACHE_FIELDS, 1, bitidx); _mi_bitmap_unclaim(cache_inuse, MI_CACHE_FIELDS, 1, bitidx);
@ -90,27 +89,26 @@ mi_decl_noinline void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* comm
static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats) static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, void* p, size_t total, mi_stats_t* stats)
{ {
if (mi_commit_mask_is_empty(*cmask)) { if (mi_commit_mask_is_empty(cmask)) {
// nothing // nothing
} }
else if (mi_commit_mask_is_full(*cmask)) { else if (mi_commit_mask_is_full(cmask)) {
_mi_os_decommit(p, total, stats); _mi_os_decommit(p, total, stats);
} }
else { else {
// todo: one call to decommit the whole at once? // todo: one call to decommit the whole at once?
mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0); mi_assert_internal((total%MI_COMMIT_MASK_BITS)==0);
size_t part = total/MI_COMMIT_MASK_BITS; size_t part = total/MI_COMMIT_MASK_BITS;
uintptr_t idx; ptrdiff_t idx;
uintptr_t count; ptrdiff_t count;
mi_commit_mask_t mask = *cmask; mi_commit_mask_foreach(cmask, idx, count) {
mi_commit_mask_foreach(mask, idx, count) {
void* start = (uint8_t*)p + (idx*part); void* start = (uint8_t*)p + (idx*part);
size_t size = count*part; size_t size = count*part;
_mi_os_decommit(start, size, stats); _mi_os_decommit(start, size, stats);
} }
mi_commit_mask_foreach_end() mi_commit_mask_foreach_end()
} }
*cmask = mi_commit_mask_empty(); mi_commit_mask_create_empty(cmask);
} }
#define MI_MAX_PURGE_PER_PUSH (4) #define MI_MAX_PURGE_PER_PUSH (4)
@ -135,7 +133,7 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld)
if (expire != 0 && now >= expire) { // safe read if (expire != 0 && now >= expire) { // safe read
// still expired, decommit it // still expired, decommit it
mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
mi_assert_internal(!mi_commit_mask_is_empty(slot->commit_mask) && _mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx)); mi_assert_internal(!mi_commit_mask_is_empty(&slot->commit_mask) && _mi_bitmap_is_claimed(cache_available_large, MI_CACHE_FIELDS, 1, bitidx));
_mi_abandoned_await_readers(); // wait until safe to decommit _mi_abandoned_await_readers(); // wait until safe to decommit
// decommit committed parts // decommit committed parts
// TODO: instead of decommit, we could also free to the OS? // TODO: instead of decommit, we could also free to the OS?
@ -148,7 +146,7 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld)
} }
} }
mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, mi_commit_mask_t commit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld) mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld)
{ {
#ifdef MI_CACHE_DISABLE #ifdef MI_CACHE_DISABLE
return false; return false;
@ -187,7 +185,7 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me
slot->memid = memid; slot->memid = memid;
slot->is_pinned = is_pinned; slot->is_pinned = is_pinned;
mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0); mi_atomic_storei64_relaxed(&slot->expire,(mi_msecs_t)0);
slot->commit_mask = commit_mask; slot->commit_mask = *commit_mask;
if (!mi_commit_mask_is_empty(commit_mask) && !is_large && !is_pinned && mi_option_is_enabled(mi_option_allow_decommit)) { if (!mi_commit_mask_is_empty(commit_mask) && !is_large && !is_pinned && mi_option_is_enabled(mi_option_allow_decommit)) {
long delay = mi_option_get(mi_option_segment_decommit_delay); long delay = mi_option_get(mi_option_segment_decommit_delay);
if (delay == 0) { if (delay == 0) {

135
src/segment.c
View File

@ -127,7 +127,7 @@ static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(segment->abandoned <= segment->used); mi_assert_internal(segment->abandoned <= segment->used);
mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id()); mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id());
mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask)); // can only decommit committed blocks mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask)); // can only decommit committed blocks
//mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0); //mi_assert_internal(segment->segment_info_size % MI_SEGMENT_SLICE_SIZE == 0);
mi_slice_t* slice = &segment->slices[0]; mi_slice_t* slice = &segment->slices[0];
const mi_slice_t* end = mi_segment_slices_end(segment); const mi_slice_t* end = mi_segment_slices_end(segment);
@ -256,8 +256,8 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) {
// _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats); // _mi_os_free(segment, mi_segment_size(segment), /*segment->memid,*/ tld->stats);
const size_t size = mi_segment_size(segment); const size_t size = mi_segment_size(segment);
if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, segment->commit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) { if (size != MI_SEGMENT_SIZE || !_mi_segment_cache_push(segment, size, segment->memid, &segment->commit_mask, segment->mem_is_large, segment->mem_is_pinned, tld->os)) {
const size_t csize = mi_commit_mask_committed_size(segment->commit_mask, size); const size_t csize = mi_commit_mask_committed_size(&segment->commit_mask, size);
if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize); if (csize > 0 && !segment->mem_is_pinned) _mi_stat_decrease(&_mi_stats_main.committed, csize);
_mi_abandoned_await_readers(); // wait until safe to free _mi_abandoned_await_readers(); // wait until safe to free
_mi_arena_free(segment, mi_segment_size(segment), segment->memid, segment->mem_is_pinned /* pretend not committed to not double count decommits */, tld->os); _mi_arena_free(segment, mi_segment_size(segment), segment->memid, segment->mem_is_pinned /* pretend not committed to not double count decommits */, tld->os);
@ -331,74 +331,85 @@ void _mi_segment_thread_collect(mi_segments_tld_t* tld) {
Span management Span management
----------------------------------------------------------- */ ----------------------------------------------------------- */
static mi_commit_mask_t mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size) { static ptrdiff_t _mi_aligni_up(ptrdiff_t sz, size_t alignment) {
mi_assert_internal(_mi_ptr_segment(p) == segment); return (ptrdiff_t)_mi_align_up(sz, alignment);
if (size == 0 || size > MI_SEGMENT_SIZE) return 0; }
if (p >= (uint8_t*)segment + mi_segment_size(segment)) return 0;
uintptr_t diff = (p - (uint8_t*)segment); static ptrdiff_t _mi_aligni_down(ptrdiff_t sz, size_t alignment) {
uintptr_t start; return (ptrdiff_t)_mi_align_down(sz, alignment);
uintptr_t end; }
static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size, mi_commit_mask_t* cm) {
mi_assert_internal(_mi_ptr_segment(p) == segment);
mi_commit_mask_create_empty(cm);
if (size == 0 || size > MI_SEGMENT_SIZE) return;
if (p >= (uint8_t*)segment + mi_segment_size(segment)) return;
ptrdiff_t diff = (p - (uint8_t*)segment);
ptrdiff_t start;
ptrdiff_t end;
if (conservative) { if (conservative) {
start = _mi_align_up(diff, MI_COMMIT_SIZE); start = _mi_aligni_up(diff, MI_COMMIT_SIZE);
end = _mi_align_down(diff + size, MI_COMMIT_SIZE); end = _mi_aligni_down(diff + size, MI_COMMIT_SIZE);
} }
else { else {
start = _mi_align_down(diff, MI_COMMIT_SIZE); start = _mi_aligni_down(diff, MI_COMMIT_SIZE);
end = _mi_align_up(diff + size, MI_COMMIT_SIZE); end = _mi_aligni_up(diff + size, MI_COMMIT_SIZE);
} }
mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0); mi_assert_internal(start % MI_COMMIT_SIZE==0 && end % MI_COMMIT_SIZE == 0);
*start_p = (uint8_t*)segment + start; *start_p = (uint8_t*)segment + start;
*full_size = (end > start ? end - start : 0); *full_size = (end > start ? end - start : 0);
if (*full_size == 0) return 0; if (*full_size == 0) return;
uintptr_t bitidx = start / MI_COMMIT_SIZE; ptrdiff_t bitidx = start / MI_COMMIT_SIZE;
mi_assert_internal(bitidx < (MI_INTPTR_SIZE*8)); mi_assert_internal(bitidx < MI_COMMIT_MASK_BITS);
uintptr_t bitcount = *full_size / MI_COMMIT_SIZE; // can be 0 ptrdiff_t bitcount = *full_size / MI_COMMIT_SIZE; // can be 0
if (bitidx + bitcount > MI_INTPTR_SIZE*8) { if (bitidx + bitcount > MI_COMMIT_MASK_BITS) {
_mi_warning_message("commit mask overflow: %zu %zu %zu %zu 0x%p %zu\n", bitidx, bitcount, start, end, p, size); _mi_warning_message("commit mask overflow: %zu %zu %zu %zu 0x%p %zu\n", bitidx, bitcount, start, end, p, size);
} }
mi_assert_internal((bitidx + bitcount) <= (MI_INTPTR_SIZE*8)); mi_assert_internal((bitidx + bitcount) <= MI_COMMIT_MASK_BITS);
mi_commit_mask_create(bitidx, bitcount, cm);
return mi_commit_mask_create(bitidx, bitcount);
} }
static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, size_t size, mi_stats_t* stats) { static bool mi_segment_commitx(mi_segment_t* segment, bool commit, uint8_t* p, size_t size, mi_stats_t* stats) {
// commit liberal, but decommit conservative // commit liberal, but decommit conservative
uint8_t* start; uint8_t* start;
size_t full_size; size_t full_size;
mi_commit_mask_t mask = mi_segment_commit_mask(segment,!commit/*conservative*/,p,size,&start,&full_size); mi_commit_mask_t mask;
if (mi_commit_mask_is_empty(mask) || full_size==0) return true; mi_segment_commit_mask(segment, !commit/*conservative*/, p, size, &start, &full_size, &mask);
if (mi_commit_mask_is_empty(&mask) || full_size==0) return true;
if (commit && !mi_commit_mask_all_set(segment->commit_mask, mask)) { if (commit && !mi_commit_mask_all_set(&segment->commit_mask, &mask)) {
bool is_zero = false; bool is_zero = false;
mi_commit_mask_t cmask = mi_commit_mask_intersect(segment->commit_mask, mask); mi_commit_mask_t cmask;
_mi_stat_decrease(&_mi_stats_main.committed, mi_commit_mask_committed_size(cmask, MI_SEGMENT_SIZE)); // adjust for overlap mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_decrease(&_mi_stats_main.committed, mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false; if (!_mi_os_commit(start,full_size,&is_zero,stats)) return false;
mi_commit_mask_set(&segment->commit_mask,mask); mi_commit_mask_set(&segment->commit_mask, &mask);
} }
else if (!commit && mi_commit_mask_any_set(segment->commit_mask,mask)) { else if (!commit && mi_commit_mask_any_set(&segment->commit_mask, &mask)) {
mi_assert_internal((void*)start != (void*)segment); mi_assert_internal((void*)start != (void*)segment);
mi_commit_mask_t cmask = mi_commit_mask_intersect(segment->commit_mask, mask); mi_commit_mask_t cmask;
_mi_stat_increase(&_mi_stats_main.committed, full_size - mi_commit_mask_committed_size(cmask, MI_SEGMENT_SIZE)); // adjust for overlap mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask);
_mi_stat_increase(&_mi_stats_main.committed, full_size - mi_commit_mask_committed_size(&cmask, MI_SEGMENT_SIZE)); // adjust for overlap
if (segment->allow_decommit) { _mi_os_decommit(start, full_size, stats); } // ok if this fails if (segment->allow_decommit) { _mi_os_decommit(start, full_size, stats); } // ok if this fails
mi_commit_mask_clear(&segment->commit_mask, mask); mi_commit_mask_clear(&segment->commit_mask, &mask);
} }
// increase expiration of reusing part of the delayed decommit // increase expiration of reusing part of the delayed decommit
if (commit && mi_commit_mask_any_set(segment->decommit_mask, mask)) { if (commit && mi_commit_mask_any_set(&segment->decommit_mask, &mask)) {
segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay); segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay);
} }
// always undo delayed decommits // always undo delayed decommits
mi_commit_mask_clear(&segment->decommit_mask, mask); mi_commit_mask_clear(&segment->decommit_mask, &mask);
mi_assert_internal((segment->commit_mask & segment->decommit_mask) == segment->decommit_mask); mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask));
return true; return true;
} }
static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) {
mi_assert_internal(mi_commit_mask_all_set(segment->commit_mask, segment->decommit_mask)); mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->decommit_mask));
if (mi_commit_mask_is_full(segment->commit_mask) && mi_commit_mask_is_empty(segment->decommit_mask)) return true; // fully committed if (mi_commit_mask_is_full(&segment->commit_mask) && mi_commit_mask_is_empty(&segment->decommit_mask)) return true; // fully committed
return mi_segment_commitx(segment,true,p,size,stats); return mi_segment_commitx(segment,true,p,size,stats);
} }
@ -411,27 +422,30 @@ static void mi_segment_perhaps_decommit(mi_segment_t* segment, uint8_t* p, size_
// register for future decommit in the decommit mask // register for future decommit in the decommit mask
uint8_t* start; uint8_t* start;
size_t full_size; size_t full_size;
mi_commit_mask_t mask = mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size); mi_commit_mask_t mask;
if (mi_commit_mask_is_empty(mask) || full_size==0) return; mi_segment_commit_mask(segment, true /*conservative*/, p, size, &start, &full_size, &mask);
if (mi_commit_mask_is_empty(&mask) || full_size==0) return;
// update delayed commit // update delayed commit
mi_commit_mask_set(&segment->decommit_mask, mi_commit_mask_intersect(mask,segment->commit_mask)); // only decommit what is committed; span_free may try to decommit more mi_commit_mask_t cmask;
mi_commit_mask_create_intersect(&segment->commit_mask, &mask, &cmask); // only decommit what is committed; span_free may try to decommit more
mi_commit_mask_set(&segment->decommit_mask, &cmask);
segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay); segment->decommit_expire = _mi_clock_now() + mi_option_get(mi_option_reset_delay);
} }
} }
static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_stats_t* stats) { static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_stats_t* stats) {
if (!segment->allow_decommit || mi_commit_mask_is_empty(segment->decommit_mask)) return; if (!segment->allow_decommit || mi_commit_mask_is_empty(&segment->decommit_mask)) return;
mi_msecs_t now = _mi_clock_now(); mi_msecs_t now = _mi_clock_now();
if (!force && now < segment->decommit_expire) return; if (!force && now < segment->decommit_expire) return;
mi_commit_mask_t mask = segment->decommit_mask; mi_commit_mask_t mask = segment->decommit_mask;
segment->decommit_expire = 0; segment->decommit_expire = 0;
segment->decommit_mask = mi_commit_mask_empty(); mi_commit_mask_create_empty(&segment->decommit_mask);
uintptr_t idx; ptrdiff_t idx;
uintptr_t count; ptrdiff_t count;
mi_commit_mask_foreach(mask, idx, count) { mi_commit_mask_foreach(&mask, idx, count) {
// if found, decommit that sequence // if found, decommit that sequence
if (count > 0) { if (count > 0) {
uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE); uint8_t* p = (uint8_t*)segment + (idx*MI_COMMIT_SIZE);
@ -439,8 +453,7 @@ static void mi_segment_delayed_decommit(mi_segment_t* segment, bool force, mi_st
mi_segment_commitx(segment, false, p, size, stats); mi_segment_commitx(segment, false, p, size, stats);
} }
} }
mi_commit_mask_foreach_end() mi_commit_mask_foreach_end()
mi_assert_internal(mi_commit_mask_is_empty(segment->decommit_mask));
} }
@ -649,7 +662,14 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
// Try to get from our cache first // Try to get from our cache first
bool is_zero = false; bool is_zero = false;
const bool commit_info_still_good = (segment != NULL); const bool commit_info_still_good = (segment != NULL);
mi_commit_mask_t commit_mask = (segment != NULL ? segment->commit_mask : mi_commit_mask_empty()); mi_commit_mask_t commit_mask;
if (segment != NULL) {
commit_mask = segment->commit_mask;
}
else {
mi_commit_mask_create_empty(&commit_mask);
}
if (segment==NULL) { if (segment==NULL) {
// Allocate the segment from the OS // Allocate the segment from the OS
bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy bool mem_large = (!eager_delay && (MI_SECURE==0)); // only allow large OS pages once we are no longer lazy
@ -659,23 +679,30 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
if (segment==NULL) { if (segment==NULL) {
segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld); segment = (mi_segment_t*)_mi_arena_alloc_aligned(segment_size, MI_SEGMENT_SIZE, &commit, &mem_large, &is_pinned, &is_zero, &memid, os_tld);
if (segment == NULL) return NULL; // failed to allocate if (segment == NULL) return NULL; // failed to allocate
commit_mask = (commit ? mi_commit_mask_full() : mi_commit_mask_empty()); if (commit) {
mi_commit_mask_create_full(&commit_mask);
}
else {
mi_commit_mask_create_empty(&commit_mask);
}
} }
mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0); mi_assert_internal(segment != NULL && (uintptr_t)segment % MI_SEGMENT_SIZE == 0);
const size_t commit_needed = _mi_divide_up(info_slices*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE); const size_t commit_needed = _mi_divide_up(info_slices*MI_SEGMENT_SLICE_SIZE, MI_COMMIT_SIZE);
mi_assert_internal(commit_needed>0); mi_assert_internal(commit_needed>0);
if (!mi_commit_mask_all_set(commit_mask,mi_commit_mask_create(0, commit_needed))) { mi_commit_mask_t commit_needed_mask;
mi_commit_mask_create(0, commit_needed, &commit_needed_mask);
if (!mi_commit_mask_all_set(&commit_mask, &commit_needed_mask)) {
// at least commit the info slices // at least commit the info slices
mi_assert_internal(commit_needed*MI_COMMIT_SIZE > info_slices*MI_SEGMENT_SLICE_SIZE); mi_assert_internal(commit_needed*MI_COMMIT_SIZE >= info_slices*MI_SEGMENT_SLICE_SIZE);
bool ok = _mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, &is_zero, tld->stats); bool ok = _mi_os_commit(segment, commit_needed*MI_COMMIT_SIZE, &is_zero, tld->stats);
if (!ok) return NULL; // failed to commit if (!ok) return NULL; // failed to commit
mi_commit_mask_set(&commit_mask,mi_commit_mask_create(0, commit_needed)); mi_commit_mask_set(&commit_mask, &commit_needed_mask);
} }
segment->memid = memid; segment->memid = memid;
segment->mem_is_pinned = is_pinned; segment->mem_is_pinned = is_pinned;
segment->mem_is_large = mem_large; segment->mem_is_large = mem_large;
segment->mem_is_committed = mi_commit_mask_is_full(commit_mask); segment->mem_is_committed = mi_commit_mask_is_full(&commit_mask);
mi_segments_track_size((long)(segment_size), tld); mi_segments_track_size((long)(segment_size), tld);
_mi_segment_map_allocated_at(segment); _mi_segment_map_allocated_at(segment);
} }
@ -692,7 +719,7 @@ static mi_segment_t* mi_segment_init(mi_segment_t* segment, size_t required, mi_
segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed segment->commit_mask = commit_mask; // on lazy commit, the initial part is always committed
segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_pinned && !segment->mem_is_large); segment->allow_decommit = (mi_option_is_enabled(mi_option_allow_decommit) && !segment->mem_is_pinned && !segment->mem_is_large);
segment->decommit_expire = 0; segment->decommit_expire = 0;
segment->decommit_mask = mi_commit_mask_empty(); mi_commit_mask_create_empty( &segment->decommit_mask );
} }
// initialize segment info // initialize segment info

4
test/test-stress.c
View File

@ -39,12 +39,12 @@ static size_t use_one_size = 0; // use single object size of `N * s
// #define USE_STD_MALLOC // #define USE_STD_MALLOC
#ifdef USE_STD_MALLOC #ifdef USE_STD_MALLOC
#define custom_calloc(n,s) calloc(n,s) #define custom_calloc(n,s) malloc(n*s)
#define custom_realloc(p,s) realloc(p,s) #define custom_realloc(p,s) realloc(p,s)
#define custom_free(p) free(p) #define custom_free(p) free(p)
#else #else
#include <mimalloc.h> #include <mimalloc.h>
#define custom_calloc(n,s) mi_calloc(n,s) #define custom_calloc(n,s) mi_malloc(n*s)
#define custom_realloc(p,s) mi_realloc(p,s) #define custom_realloc(p,s) mi_realloc(p,s)
#define custom_free(p) mi_free(p) #define custom_free(p) mi_free(p)
#endif #endif