From bd2ac3c92e3e00ec02b09081a7678478b42abf65 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Feb 2022 16:17:21 -0800 Subject: [PATCH] collect segment cache on mi_collect --- include/mimalloc-internal.h | 1 + src/heap.c | 5 ++++- src/segment-cache.c | 15 ++++++++++----- test/test-stress.c | 8 +++++--- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 45775df4..88142197 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -95,6 +95,7 @@ void _mi_arena_free(void* p, size_t size, size_t memid, bool is_committed, // "segment-cache.c" void* _mi_segment_cache_pop(size_t size, mi_commit_mask_t* commit_mask, mi_commit_mask_t* decommit_mask, bool* large, bool* is_pinned, bool* is_zero, size_t* memid, mi_os_tld_t* tld); bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld); +void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld); void _mi_segment_map_allocated_at(const mi_segment_t* segment); void _mi_segment_map_freed_at(const mi_segment_t* segment); diff --git a/src/heap.c b/src/heap.c index d7975b0b..416a9a8d 100644 --- a/src/heap.c +++ b/src/heap.c @@ -147,11 +147,14 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); - // collect segment caches + // collect segment local caches if (collect >= MI_FORCE) { _mi_segment_thread_collect(&heap->tld->segments); } + // decommit in global segment caches + _mi_segment_cache_collect( collect >= MI_FORCE, &heap->tld->os); + // collect regions on program-exit (or shared library unload) if (collect >= MI_FORCE && _mi_is_main_thread() && mi_heap_is_backing(heap)) { //_mi_mem_collect(&heap->tld->os); diff --git a/src/segment-cache.c b/src/segment-cache.c index cabdec8f..41d71e5e 100644 --- a/src/segment-cache.c +++ b/src/segment-cache.c @@ -115,13 +115,14 @@ static mi_decl_noinline void mi_commit_mask_decommit(mi_commit_mask_t* cmask, vo #define MI_MAX_PURGE_PER_PUSH (4) -static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld) +static mi_decl_noinline void mi_segment_cache_purge(bool force, mi_os_tld_t* tld) { MI_UNUSED(tld); mi_msecs_t now = _mi_clock_now(); - size_t idx = (_mi_random_shuffle((uintptr_t)now) % MI_CACHE_MAX); // random start size_t purged = 0; - for (size_t visited = 0; visited < MI_CACHE_FIELDS; visited++,idx++) { // probe just N slots + const size_t max_visits = (force ? MI_CACHE_MAX /* visit all */ : MI_CACHE_FIELDS /* probe at most N (=16) slots */); + size_t idx = (force ? 0 : _mi_random_shuffle((uintptr_t)now) % MI_CACHE_MAX /* random start */ ); + for (size_t visited = 0; visited < max_visits; visited++,idx++) { // visit N slots if (idx >= MI_CACHE_MAX) idx = 0; // wrap mi_cache_slot_t* slot = &cache[idx]; mi_msecs_t expire = mi_atomic_loadi64_relaxed(&slot->expire); @@ -144,11 +145,15 @@ static mi_decl_noinline void mi_segment_cache_purge(mi_os_tld_t* tld) } _mi_bitmap_unclaim(cache_available, MI_CACHE_FIELDS, 1, bitidx); // make it available again for a pop } - if (purged > MI_MAX_PURGE_PER_PUSH) break; // bound to no more than N purge tries per push + if (!force && purged > MI_MAX_PURGE_PER_PUSH) break; // bound to no more than N purge tries per push } } } +void _mi_segment_cache_collect(bool force, mi_os_tld_t* tld) { + mi_segment_cache_purge(force, tld ); +} + mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t memid, const mi_commit_mask_t* commit_mask, const mi_commit_mask_t* decommit_mask, bool is_large, bool is_pinned, mi_os_tld_t* tld) { #ifdef MI_CACHE_DISABLE @@ -167,7 +172,7 @@ mi_decl_noinline bool _mi_segment_cache_push(void* start, size_t size, size_t me } // purge expired entries - mi_segment_cache_purge(tld); + mi_segment_cache_purge(false /* force? */, tld); // find an available slot mi_bitmap_index_t bitidx; diff --git a/test/test-stress.c b/test/test-stress.c index 100c6d66..ff5fffeb 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -189,11 +189,13 @@ static void test_stress(void) { free_items(p); } } + #ifndef NDEBUG //mi_collect(false); - //mi_debug_show_arenas(); -#if !defined(NDEBUG) || defined(MI_TSAN) + //mi_debug_show_arenas(); + #endif + #if !defined(NDEBUG) || defined(MI_TSAN) if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); } -#endif + #endif } }