From bf5932c3c6ca52184690f88f0d4815c92a3ad42b Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 19 Apr 2024 12:43:05 -0700 Subject: [PATCH] use better purging for segments_collect --- include/mimalloc/internal.h | 4 ++-- src/arena.c | 4 ++-- src/heap.c | 9 ++++----- src/segment.c | 14 +++++++------- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 47aa46ff..c944a126 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -124,7 +124,7 @@ void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); bool _mi_arena_contains(const void* p); -void _mi_arena_collect(bool force_purge, mi_stats_t* stats); +void _mi_arenas_collect(bool force_purge, mi_stats_t* stats); void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); bool _mi_arena_segment_clear_abandoned(mi_segment_t* segment); @@ -155,7 +155,7 @@ void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); #endif -void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld); +void _mi_segments_collect(bool force, mi_segments_tld_t* tld); void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); void _mi_abandoned_await_readers(void); bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment); diff --git a/src/arena.c b/src/arena.c index b78d69f8..6e5e21b7 100644 --- a/src/arena.c +++ b/src/arena.c @@ -708,7 +708,7 @@ static void mi_arenas_unsafe_destroy(void) { } // Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired -void _mi_arena_collect(bool force_purge, mi_stats_t* stats) { +void _mi_arenas_collect(bool force_purge, mi_stats_t* stats) { mi_arenas_try_purge(force_purge, force_purge /* visit all? */, stats); } @@ -716,7 +716,7 @@ void _mi_arena_collect(bool force_purge, mi_stats_t* stats) { // for dynamic libraries that are unloaded and need to release all their allocated memory. void _mi_arena_unsafe_destroy_all(mi_stats_t* stats) { mi_arenas_unsafe_destroy(); - _mi_arena_collect(true /* force purge */, stats); // purge non-owned arenas + _mi_arenas_collect(true /* force purge */, stats); // purge non-owned arenas } // Is a pointer inside any of our arenas? diff --git a/src/heap.c b/src/heap.c index a878f32b..f7f6c8e8 100644 --- a/src/heap.c +++ b/src/heap.c @@ -104,10 +104,6 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t // still used blocks but the thread is done; abandon the page _mi_page_abandon(page, pq); } - if (collect == MI_FORCE) { - mi_segment_t* segment = _mi_page_segment(page); - _mi_segment_collect(segment, true /* force? */, &heap->tld->segments); - } return true; // don't break } @@ -157,13 +153,16 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL ); + // collect segments (purge pages, this can be expensive so don't force on abandonment) + _mi_segments_collect(collect == MI_FORCE, &heap->tld->segments); + // if forced, collect thread data cache on program-exit (or shared library unload) if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache } // collect arenas (this is program wide so don't force purges on abandonment of threads) - _mi_arena_collect(collect == MI_FORCE /* force purge? */, &heap->tld->stats); + _mi_arenas_collect(collect == MI_FORCE /* force purge? */, &heap->tld->stats); } void _mi_heap_collect_abandon(mi_heap_t* heap) { diff --git a/src/segment.c b/src/segment.c index 8ea76bfe..4bb4aaf5 100644 --- a/src/segment.c +++ b/src/segment.c @@ -362,14 +362,14 @@ static void mi_segment_remove_all_purges(mi_segment_t* segment, bool force_purge } } -static void mi_pages_try_purge(mi_segments_tld_t* tld) { +static void mi_pages_try_purge(bool force, mi_segments_tld_t* tld) { if (mi_option_get(mi_option_purge_delay) < 0) return; // purging is not allowed mi_msecs_t now = _mi_clock_now(); mi_page_queue_t* pq = &tld->pages_purge; // from oldest up to the first that has not expired yet mi_page_t* page = pq->last; - while (page != NULL && mi_page_purge_is_expired(page,now)) { + while (page != NULL && (force || mi_page_purge_is_expired(page,now))) { mi_page_t* const prev = page->prev; // save previous field mi_page_purge_remove(page, tld); // remove from the list to maintain invariant for mi_page_purge mi_page_purge(_mi_page_segment(page), page, tld); @@ -515,9 +515,9 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se _mi_arena_free(segment, segment_size, committed_size, segment->memid, tld->stats); } -// called from `heap_collect`. This can be called per-page. -void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { - MI_UNUSED(segment); MI_UNUSED(force); MI_UNUSED(tld); +// called from `heap_collect`. +void _mi_segments_collect(bool force, mi_segments_tld_t* tld) { + mi_pages_try_purge(force,tld); } @@ -734,7 +734,7 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld) mi_assert(page != NULL); mi_segment_t* segment = _mi_page_segment(page); mi_assert_expensive(mi_segment_is_valid(segment,tld)); - mi_pages_try_purge(tld); + mi_pages_try_purge(false /*force?*/, tld); // mark it as free now mi_segment_page_clear(segment, page, tld); @@ -793,7 +793,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { // Potentially force purge. Only abandoned segments in arena memory can be // reclaimed without a free so if a segment is not from an arena we force purge here to be conservative. - mi_pages_try_purge(tld); + mi_pages_try_purge(false /*force?*/,tld); const bool force_purge = (segment->memid.memkind != MI_MEM_ARENA) || mi_option_is_enabled(mi_option_abandoned_page_purge); mi_segment_remove_all_purges(segment, force_purge, tld);