merge from dev

include/mimalloc/prim.h

@@ -47,7 +47,8 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la
 
 // Commit memory. Returns error code or 0 on success.
 // For example, on Linux this would make the memory PROT_READ|PROT_WRITE.
-int _mi_prim_commit(void* addr, size_t size);
+// `is_zero` is set to true if the memory was zero initialized (e.g. on Windows)
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero);
 
 // Decommit memory. Returns error code or 0 on success. The `needs_recommit` result is true
 // if the memory would need to be re-committed. For example, on Windows this is always true,

include/mimalloc/track.h

@@ -79,7 +79,7 @@ defined, undefined, or not accessible at all:
 // windows event tracing
 
 #define MI_TRACK_ENABLED      1
-#define MI_TRACK_HEAP_DESTROY 0
+#define MI_TRACK_HEAP_DESTROY 1
 #define MI_TRACK_TOOL         "ETW"
 
 #define WIN32_LEAN_AND_MEAN

src/alloc.c

@@ -707,14 +707,12 @@ void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero)
     if (zero && newsize > size) {
       // also set last word in the previous allocation to zero to ensure any padding is zero-initialized
       const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0);
-      memset((uint8_t*)newp + start, 0, newsize - start);
+      _mi_memzero((uint8_t*)newp + start, newsize - start);
     }
     if mi_likely(p != NULL) {
-      if mi_likely(_mi_is_aligned(p, sizeof(uintptr_t))) {  // a client may pass in an arbitrary pointer `p`..
-        const size_t copysize = (newsize > size ? size : newsize);
-        mi_track_mem_defined(p,copysize);  // _mi_useable_size may be too large for byte precise memory tracking..
-        _mi_memcpy_aligned(newp, p, copysize);
-      }
+      const size_t copysize = (newsize > size ? size : newsize);
+      mi_track_mem_defined(p,copysize);  // _mi_useable_size may be too large for byte precise memory tracking..
+      _mi_memcpy(newp, p, copysize);
       mi_free(p); // only free the original pointer if successful
     }
   }

src/os.c

@@ -394,12 +394,17 @@ bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats
   if (csize == 0) return true;
 
   // commit  
-  int err = _mi_prim_commit(start, csize); 
+  bool os_is_zero = false;
+  int err = _mi_prim_commit(start, csize, &os_is_zero); 
   if (err != 0) {
     _mi_warning_message("cannot commit OS memory (error: %d (0x%x), address: %p, size: 0x%zx bytes)\n", err, err, start, csize);
+    return false;
   }
-  mi_assert_internal(err == 0);
-  return (err == 0);
+  if (os_is_zero && is_zero != NULL) { 
+    *is_zero = true;
+    mi_assert_expensive(mi_mem_is_zero(start, csize));
+  } 
+  return true;
 }
 
 static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_stats_t* tld_stats) {

src/prim/unix/prim.c

@@ -342,8 +342,9 @@ static void unix_mprotect_hint(int err) {
   #endif
 }
 
-int _mi_prim_commit(void* start, size_t size) {
+int _mi_prim_commit(void* start, size_t size, bool* is_zero) {
   // commit: ensure we can access the area
+  *is_zero = false;
   int err = mprotect(start, size, (PROT_READ | PROT_WRITE));
   if (err != 0) { err = errno; }  
   unix_mprotect_hint(err);

src/prim/wasi/prim.c

@@ -128,8 +128,9 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la
 // Commit/Reset/Protect
 //---------------------------------------------
 
-int _mi_prim_commit(void* addr, size_t size) {
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
   MI_UNUSED(addr); MI_UNUSED(size); 
+  *is_zero = false;
   return 0;
 }
 

src/prim/windows/prim.c

@@ -259,14 +259,26 @@ int _mi_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_la
 #pragma warning(disable:6250)   // suppress warning calling VirtualFree without MEM_RELEASE (for decommit)
 #endif
 
-int _mi_prim_commit(void* addr, size_t size) {
+int _mi_prim_commit(void* addr, size_t size, bool* is_zero) {
+  *is_zero = false;
+  /*
+  // zero'ing only happens on an initial commit... but checking upfront seems expensive..
+  _MEMORY_BASIC_INFORMATION meminfo; _mi_memzero_var(meminfo);
+  if (VirtualQuery(addr, &meminfo, size) > 0) {
+    if ((meminfo.State & MEM_COMMIT) == 0) {
+      *is_zero = true;
+    }
+  }
+  */
+  // commit
   void* p = VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
-  return (p == addr ? 0 : (int)GetLastError());  
+  if (p == NULL) return (int)GetLastError();
+  return 0;
 }
 
 int _mi_prim_decommit(void* addr, size_t size, bool* needs_recommit) {  
   BOOL ok = VirtualFree(addr, size, MEM_DECOMMIT);
-  *needs_recommit = true;  // for safetly, assume always decommitted even in the case of an error.
+  *needs_recommit = true;  // for safety, assume always decommitted even in the case of an error.
   return (ok ? 0 : (int)GetLastError());
 }
 
@@ -274,11 +286,11 @@ int _mi_prim_reset(void* addr, size_t size) {
   void* p = VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
   mi_assert_internal(p == addr);
   #if 1
-  if (p == addr && addr != NULL) {
+  if (p != NULL) {
     VirtualUnlock(addr,size); // VirtualUnlock after MEM_RESET removes the memory from the working set
   }
   #endif
-  return (p == addr ? 0 : (int)GetLastError());
+  return (p != NULL ? 0 : (int)GetLastError());
 }
 
 int _mi_prim_protect(void* addr, size_t size, bool protect) {

src/segment.c

@@ -1579,9 +1579,12 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_bloc
   mi_assert_internal(page->used == 1); // this is called just before the free
   mi_assert_internal(page->free == NULL);
   if (segment->allow_decommit) {
-    const size_t csize = mi_usable_size(block) - sizeof(mi_block_t);
-    uint8_t* p = (uint8_t*)block + sizeof(mi_block_t);
-    _mi_os_reset(p, csize, &_mi_stats_main);  // note: cannot use segment_decommit on huge segments
+    size_t csize = mi_usable_size(block);
+    if (csize > sizeof(mi_block_t)) {
+      csize = csize - sizeof(mi_block_t);
+      uint8_t* p = (uint8_t*)block + sizeof(mi_block_t);
+      _mi_os_reset(p, csize, &_mi_stats_main);  // note: cannot use segment_decommit on huge segments
+    }
   }
 }
 #endif