merge from dev

include/mimalloc.h

@@ -266,12 +266,13 @@ typedef enum mi_option_e {
   mi_option_verbose,
   // the following options are experimental
   mi_option_eager_commit,
+  mi_option_eager_region_commit,
+  mi_option_reset_decommits,
   mi_option_large_os_pages,         // implies eager commit
   mi_option_reserve_huge_os_pages,
   mi_option_segment_cache,
   mi_option_page_reset,
   mi_option_segment_reset,
-  mi_option_reset_decommits,
   mi_option_eager_commit_delay,
   mi_option_allow_decommit,
   mi_option_reset_delay,

src/options.c

@@ -56,16 +56,22 @@ static mi_option_desc_t options[_mi_option_last] =
   { 0, UNINIT, MI_OPTION(verbose) },
 
   // the following options are experimental and not all combinations make sense.
-  { 0, UNINIT, MI_OPTION(eager_commit) },        // note: needs to be on when eager_region_commit is enabled
+  { 1, UNINIT, MI_OPTION(eager_commit) },        // commit on demand
+  #if defined(_WIN32) || (MI_INTPTR_SIZE <= 4)   // and other OS's without overcommit?
+  { 0, UNINIT, MI_OPTION(eager_region_commit) },
+  { 1, UNINIT, MI_OPTION(reset_decommits) },     // reset decommits memory
+  #else
+  { 1, UNINIT, MI_OPTION(eager_region_commit) }, 
+  { 0, UNINIT, MI_OPTION(reset_decommits) },     // reset uses MADV_FREE/MADV_DONTNEED
+  #endif
   { 0, UNINIT, MI_OPTION(large_os_pages) },      // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
   { 0, UNINIT, MI_OPTION(reserve_huge_os_pages) },
   { 0, UNINIT, MI_OPTION(segment_cache) },       // cache N segments per thread
   { 0, UNINIT, MI_OPTION(page_reset) },          // reset pages on free
   { 0, UNINIT, MI_OPTION(segment_reset) },       // reset segment memory on free (needs eager commit)
-  { 1, UNINIT, MI_OPTION(reset_decommits) },     // reset decommits memory
   { 0, UNINIT, MI_OPTION(eager_commit_delay) },  // the first N segments per thread are not eagerly committed
   { 0, UNINIT, MI_OPTION(allow_decommit) },      // decommit pages when not eager committed
-  { 1000, UNINIT, MI_OPTION(reset_delay) },       // reset delay in milli-seconds
+  { 100,  UNINIT, MI_OPTION(reset_delay) },       // reset delay in milli-seconds
   { 1000, UNINIT, MI_OPTION(arena_reset_delay) }, // reset delay in milli-seconds
   { 0,    UNINIT, MI_OPTION(use_numa_nodes) },    // 0 = use available numa nodes, otherwise use at most N nodes. 
   { 100,  UNINIT, MI_OPTION(os_tag) },            // only apple specific for now but might serve more or less related purpose

src/os.c

@@ -602,7 +602,7 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t*
 // (but not for the reset version where we want commit to be conservative as well)
 static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservative, bool* is_zero, mi_stats_t* stats) {
   // page align in the range, commit liberally, decommit conservative
-  *is_zero = false;
+  if (is_zero != NULL) { *is_zero = false; }
   size_t csize;
   void* start = mi_os_page_align_areax(conservative, addr, size, &csize);
   if (csize == 0) return true;  // || _mi_os_is_huge_reserved(addr))
@@ -631,7 +631,7 @@ static bool mi_os_commitx(void* addr, size_t size, bool commit, bool conservativ
   #elif defined(MAP_FIXED)
   if (!commit) {
     // use mmap with MAP_FIXED to discard the existing memory (and reduce commit charge)
-    void* p = mmap(start, size, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
+    void* p = mmap(start, csize, PROT_NONE, (MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE), -1, 0);
     if (p != start) { err = errno; }
   }
   else {

src/page.c

@@ -35,7 +35,7 @@ static inline mi_block_t* mi_page_block_at(const mi_page_t* page, void* page_sta
   return (mi_block_t*)((uint8_t*)page_start + (i * page->block_size));
 }
 
-static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_stats_t* stats);
+static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t size, mi_tld_t* tld);
 
 
 #if (MI_DEBUG>=3)
@@ -244,7 +244,7 @@ static mi_page_t* mi_page_fresh_alloc(mi_heap_t* heap, mi_page_queue_t* pq, size
   mi_page_t* page = _mi_segment_page_alloc(block_size, &heap->tld->segments, &heap->tld->os);
   if (page == NULL) return NULL;
   mi_assert_internal(pq==NULL || _mi_page_segment(page)->kind != MI_SEGMENT_HUGE);
-  mi_page_init(heap, page, block_size, &heap->tld->stats);
+  mi_page_init(heap, page, block_size, heap->tld);
   _mi_stat_increase( &heap->tld->stats.pages, 1);
   if (pq!=NULL) mi_page_queue_push(heap, pq, page); // huge pages use pq==NULL
   mi_assert_expensive(_mi_page_is_valid(page));
@@ -544,8 +544,8 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co
 // Note: we also experimented with "bump" allocation on the first
 // allocations but this did not speed up any benchmark (due to an
 // extra test in malloc? or cache effects?)
-static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* stats) {
-  UNUSED(stats);
+static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_tld_t* tld) {
+  UNUSED(tld); 
   mi_assert_expensive(mi_page_is_valid_init(page));
   #if (MI_SECURE<=2)
   mi_assert(page->free == NULL);
@@ -556,7 +556,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st
 
   size_t page_size;
   _mi_page_start(_mi_page_segment(page), page, &page_size);
-  mi_stat_counter_increase(stats->pages_extended, 1);
+  mi_stat_counter_increase(tld->stats.pages_extended, 1);
 
   // calculate the extend count
   size_t extend = page->reserved - page->capacity;
@@ -574,14 +574,14 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st
 
   // and append the extend the free list
   if (extend < MI_MIN_SLICES || MI_SECURE==0) { //!mi_option_is_enabled(mi_option_secure)) {
-    mi_page_free_list_extend(page, extend, stats );
+    mi_page_free_list_extend(page, extend, &tld->stats );
   }
   else {
-    mi_page_free_list_extend_secure(heap, page, extend, stats);
+    mi_page_free_list_extend_secure(heap, page, extend, &tld->stats);
   }
   // enable the new free list
   page->capacity += (uint16_t)extend;
-  mi_stat_increase(stats->page_committed, extend * page->block_size);
+  mi_stat_increase(tld->stats.page_committed, extend * page->block_size);
 
   // extension into zero initialized memory preserves the zero'd free list
   if (!page->is_zero_init) {
@@ -591,7 +591,7 @@ static void mi_page_extend_free(mi_heap_t* heap, mi_page_t* page, mi_stats_t* st
 }
 
 // Initialize a fresh page
-static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi_stats_t* stats) {
+static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi_tld_t* tld) {
   mi_assert(page != NULL);
   mi_segment_t* segment = _mi_page_segment(page);
   mi_assert(segment != NULL);
@@ -625,7 +625,7 @@ static void mi_page_init(mi_heap_t* heap, mi_page_t* page, size_t block_size, mi
   mi_assert_expensive(mi_page_is_valid_init(page));
 
   // initialize an initial free list
-  mi_page_extend_free(heap,page,stats);
+  mi_page_extend_free(heap,page,tld);
   mi_assert(mi_page_immediate_available(page));
 }
 
@@ -670,7 +670,7 @@ static mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, mi_page_queue_t* p
 
     // 2. Try to extend
     if (page->capacity < page->reserved) {
-      mi_page_extend_free(heap, page, &heap->tld->stats);
+      mi_page_extend_free(heap, page, heap->tld);
       mi_assert_internal(mi_page_immediate_available(page));
       break;
     }
@@ -711,7 +711,7 @@ static inline mi_page_t* mi_find_free_page(mi_heap_t* heap, size_t size) {
   if (page != NULL) {
     if ((MI_SECURE >= 3) && page->capacity < page->reserved && ((_mi_heap_random(heap) & 1) == 1)) {
       // in secure mode, we extend half the time to increase randomness
-      mi_page_extend_free(heap, page, &heap->tld->stats);
+      mi_page_extend_free(heap, page, heap->tld);
       mi_assert_internal(mi_page_immediate_available(page));
     }
     else {

src/segment.c

@@ -189,7 +189,6 @@ static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
 }
 #endif
 
-
 /* -----------------------------------------------------------
  Segment size calculations
 ----------------------------------------------------------- */