diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 4ea2d855..73762711 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -149,7 +149,7 @@ mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t pag void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); -void _mi_segment_thread_collect(mi_segments_tld_t* tld); +void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld); #if MI_HUGE_PAGE_ABANDON void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); diff --git a/src/heap.c b/src/heap.c index 1a4c3fb4..587ed9a0 100644 --- a/src/heap.c +++ b/src/heap.c @@ -104,6 +104,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t // still used blocks but the thread is done; abandon the page _mi_page_abandon(page, pq); } + if (collect == MI_FORCE) { + mi_segment_t* segment = _mi_page_segment(page); + _mi_segment_collect(segment, true /* force? */, &heap->tld->segments); + } return true; // don't break } @@ -157,12 +161,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) // collect abandoned segments (in particular, purge expired parts of segments in the abandoned segment list) // note: forced purge can be quite expensive if many threads are created/destroyed so we do not force on abandonment _mi_abandoned_collect(heap, collect == MI_FORCE /* force? */, &heap->tld->segments); - - // collect segment local caches - if (force) { - _mi_segment_thread_collect(&heap->tld->segments); - } - + // if forced, collect thread data cache on program-exit (or shared library unload) if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache diff --git a/src/segment.c b/src/segment.c index e5461808..2f20421a 100644 --- a/src/segment.c +++ b/src/segment.c @@ -404,13 +404,6 @@ static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { _mi_arena_free(segment, mi_segment_size(segment), csize, segment->memid, tld->stats); } -// called by threads that are terminating -void _mi_segment_thread_collect(mi_segments_tld_t* tld) { - MI_UNUSED(tld); - // nothing to do -} - - /* ----------------------------------------------------------- Commit/Decommit ranges ----------------------------------------------------------- */ @@ -573,7 +566,7 @@ static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t } static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) { - if (!segment->allow_purge || mi_commit_mask_is_empty(&segment->purge_mask)) return; + if (!segment->allow_purge || segment->purge_expire == 0 || mi_commit_mask_is_empty(&segment->purge_mask)) return; mi_msecs_t now = _mi_clock_now(); if (!force && now < segment->purge_expire) return; @@ -595,6 +588,12 @@ static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* mi_assert_internal(mi_commit_mask_is_empty(&segment->purge_mask)); } +// called from `mi_heap_collect_ex` +// this can be called per-page so it is important that try_purge has fast exit path +void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { + MI_UNUSED(force); MI_UNUSED(tld); + mi_segment_try_purge(segment, force, tld->stats); +} /* ----------------------------------------------------------- Span free