use better purging for segments_collect

include/mimalloc/internal.h

@@ -124,7 +124,7 @@ void*      _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_
 void*      _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld);
 bool       _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id);
 bool       _mi_arena_contains(const void* p);
-void       _mi_arena_collect(bool force_purge, mi_stats_t* stats);
+void       _mi_arenas_collect(bool force_purge, mi_stats_t* stats);
 void       _mi_arena_unsafe_destroy_all(mi_stats_t* stats);
 
 bool       _mi_arena_segment_clear_abandoned(mi_segment_t* segment);
@@ -155,7 +155,7 @@ void       _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi
 void       _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block);
 #endif
 
-void       _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld);
+void       _mi_segments_collect(bool force, mi_segments_tld_t* tld);
 void       _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld);
 void       _mi_abandoned_await_readers(void);
 bool       _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment);

src/arena.c

@@ -708,7 +708,7 @@ static void mi_arenas_unsafe_destroy(void) {
 }
 
 // Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired
-void _mi_arena_collect(bool force_purge, mi_stats_t* stats) {
+void _mi_arenas_collect(bool force_purge, mi_stats_t* stats) {
   mi_arenas_try_purge(force_purge, force_purge /* visit all? */, stats);
 }
 
@@ -716,7 +716,7 @@ void _mi_arena_collect(bool force_purge, mi_stats_t* stats) {
 // for dynamic libraries that are unloaded and need to release all their allocated memory.
 void _mi_arena_unsafe_destroy_all(mi_stats_t* stats) {
   mi_arenas_unsafe_destroy();
-  _mi_arena_collect(true /* force purge */, stats);  // purge non-owned arenas
+  _mi_arenas_collect(true /* force purge */, stats);  // purge non-owned arenas
 }
 
 // Is a pointer inside any of our arenas?

src/heap.c

@@ -104,10 +104,6 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t
     // still used blocks but the thread is done; abandon the page
     _mi_page_abandon(page, pq);
   }
-  if (collect == MI_FORCE) {
-    mi_segment_t* segment = _mi_page_segment(page);
-    _mi_segment_collect(segment, true /* force? */, &heap->tld->segments);
-  }
   return true; // don't break
 }
 
@@ -157,13 +153,16 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
   mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL);
   mi_assert_internal( collect != MI_ABANDON || mi_atomic_load_ptr_acquire(mi_block_t,&heap->thread_delayed_free) == NULL );
 
+  // collect segments (purge pages, this can be expensive so don't force on abandonment)
+  _mi_segments_collect(collect == MI_FORCE, &heap->tld->segments);
+
   // if forced, collect thread data cache on program-exit (or shared library unload)
   if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
     _mi_thread_data_collect();  // collect thread data cache
   }
   
   // collect arenas (this is program wide so don't force purges on abandonment of threads)
-  _mi_arena_collect(collect == MI_FORCE /* force purge? */, &heap->tld->stats);  
+  _mi_arenas_collect(collect == MI_FORCE /* force purge? */, &heap->tld->stats);  
 }
 
 void _mi_heap_collect_abandon(mi_heap_t* heap) {

src/segment.c

@@ -362,14 +362,14 @@ static void mi_segment_remove_all_purges(mi_segment_t* segment, bool force_purge
   }
 }
 
-static void mi_pages_try_purge(mi_segments_tld_t* tld) {
+static void mi_pages_try_purge(bool force, mi_segments_tld_t* tld) {
   if (mi_option_get(mi_option_purge_delay) < 0) return;  // purging is not allowed
 
   mi_msecs_t now = _mi_clock_now();
   mi_page_queue_t* pq = &tld->pages_purge;
   // from oldest up to the first that has not expired yet
   mi_page_t* page = pq->last;
-  while (page != NULL && mi_page_purge_is_expired(page,now)) {
+  while (page != NULL && (force || mi_page_purge_is_expired(page,now))) {
     mi_page_t* const prev = page->prev; // save previous field
     mi_page_purge_remove(page, tld);    // remove from the list to maintain invariant for mi_page_purge
     mi_page_purge(_mi_page_segment(page), page, tld);
@@ -515,9 +515,9 @@ static void mi_segment_os_free(mi_segment_t* segment, size_t segment_size, mi_se
   _mi_arena_free(segment, segment_size, committed_size, segment->memid, tld->stats);
 }
 
-// called from `heap_collect`. This can be called per-page.
-void _mi_segment_collect(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) {
-  MI_UNUSED(segment); MI_UNUSED(force); MI_UNUSED(tld);
+// called from `heap_collect`. 
+void _mi_segments_collect(bool force, mi_segments_tld_t* tld) {
+  mi_pages_try_purge(force,tld);
 }
 
 
@@ -734,7 +734,7 @@ void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)
   mi_assert(page != NULL);
   mi_segment_t* segment = _mi_page_segment(page);
   mi_assert_expensive(mi_segment_is_valid(segment,tld));
-  mi_pages_try_purge(tld);
+  mi_pages_try_purge(false /*force?*/, tld);
 
   // mark it as free now
   mi_segment_page_clear(segment, page, tld);
@@ -793,7 +793,7 @@ static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) {
 
   // Potentially force purge. Only abandoned segments in arena memory can be
   // reclaimed without a free so if a segment is not from an arena we force purge here to be conservative.
-  mi_pages_try_purge(tld);
+  mi_pages_try_purge(false /*force?*/,tld);
   const bool force_purge = (segment->memid.memkind != MI_MEM_ARENA) ||  mi_option_is_enabled(mi_option_abandoned_page_purge);
   mi_segment_remove_all_purges(segment, force_purge, tld);