mirror of
https://github.com/microsoft/mimalloc.git
synced 2024-12-25 20:14:12 +08:00
Merge branch 'dev-slice'
This commit is contained in:
Daan
commit
2765ec9302
1
.gitattributes
vendored
1
.gitattributes
vendored
@ -10,4 +10,3 @@
|
||||
*.dll binary
|
||||
*.lib binary
|
||||
*.exe binary
|
||||
bin export-ignore
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -7,3 +7,5 @@ ide/vs20??/VTune*
|
||||
out/
|
||||
docs/
|
||||
*.zip
|
||||
*.tar
|
||||
*.gz
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
cmake_minimum_required(VERSION 3.13)
|
||||
cmake_minimum_required(VERSION 3.18)
|
||||
project(libmimalloc C CXX)
|
||||
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
@ -35,6 +35,7 @@ option(MI_NO_THP "Disable transparent huge pages support on Linux/And
|
||||
option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF)
|
||||
option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF)
|
||||
|
||||
include(CheckLinkerFlag) # requires cmake 3.18
|
||||
include(CheckIncludeFiles)
|
||||
include(GNUInstallDirs)
|
||||
include("cmake/mimalloc-config-version.cmake")
|
||||
@ -338,29 +339,45 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914)
|
||||
list(APPEND mi_cflags /Zc:__cplusplus)
|
||||
endif()
|
||||
|
||||
if(MINGW)
|
||||
add_definitions(-D_WIN32_WINNT=0x600)
|
||||
endif()
|
||||
|
||||
# extra needed libraries
|
||||
|
||||
# we prefer -l<lib> test over `find_library` as sometimes core libraries
|
||||
# like `libatomic` are not on the system path (see issue #898)
|
||||
function(find_link_library libname outlibname)
|
||||
check_linker_flag(C "-l${libname}" mi_has_lib${libname})
|
||||
if (mi_has_lib${libname})
|
||||
message(VERBOSE "link library: -l${libname}")
|
||||
set(${outlibname} ${libname} PARENT_SCOPE)
|
||||
else()
|
||||
find_library(MI_LIBPATH libname)
|
||||
if (MI_LIBPATH)
|
||||
message(VERBOSE "link library ${libname} at ${MI_LIBPATH}")
|
||||
set(${outlibname} ${MI_LIBPATH} PARENT_SCOPE)
|
||||
else()
|
||||
message(VERBOSE "link library not found: ${libname}")
|
||||
set(${outlibname} "" PARENT_SCOPE)
|
||||
endif()
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
if(WIN32)
|
||||
list(APPEND mi_libraries psapi shell32 user32 advapi32 bcrypt)
|
||||
set(pc_libraries "-lpsapi -lshell32 -luser32 -ladvapi32 -lbcrypt")
|
||||
else()
|
||||
set(pc_libraries "")
|
||||
find_library(MI_LIBPTHREAD pthread)
|
||||
if (MI_LIBPTHREAD)
|
||||
list(APPEND mi_libraries ${MI_LIBPTHREAD})
|
||||
set(pc_libraries "${pc_libraries} -pthread")
|
||||
find_link_library("pthread" MI_LIB_PTHREAD)
|
||||
if(MI_LIB_PTHREAD)
|
||||
list(APPEND mi_libraries "${MI_LIB_PTHREAD}")
|
||||
endif()
|
||||
find_library(MI_LIBRT rt)
|
||||
if(MI_LIBRT)
|
||||
list(APPEND mi_libraries ${MI_LIBRT})
|
||||
set(pc_libraries "${pc_libraries} -lrt")
|
||||
find_link_library("rt" MI_LIB_RT)
|
||||
if(MI_LIB_RT)
|
||||
list(APPEND mi_libraries "${MI_LIB_RT}")
|
||||
endif()
|
||||
find_library(MI_LIBATOMIC atomic)
|
||||
if (NOT MI_LIBATOMIC AND MI_USE_LIBATOMIC)
|
||||
set(MI_LIBATOMIC atomic)
|
||||
endif()
|
||||
if (MI_LIBATOMIC)
|
||||
list(APPEND mi_libraries ${MI_LIBATOMIC})
|
||||
set(pc_libraries "${pc_libraries} -latomic")
|
||||
find_link_library("atomic" MI_LIB_ATOMIC)
|
||||
if(MI_LIB_ATOMIC)
|
||||
list(APPEND mi_libraries "${MI_LIB_ATOMIC}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@ -370,6 +387,7 @@ endif()
|
||||
|
||||
# dynamic/shared library and symlinks always go to /usr/local/lib equivalent
|
||||
set(mi_install_libdir "${CMAKE_INSTALL_LIBDIR}")
|
||||
set(mi_install_bindir "${CMAKE_INSTALL_BINDIR}")
|
||||
|
||||
# static libraries and object files, includes, and cmake config files
|
||||
# are either installed at top level, or use versioned directories for side-by-side installation (default)
|
||||
@ -453,10 +471,10 @@ if(MI_BUILD_SHARED)
|
||||
add_custom_command(TARGET mimalloc POST_BUILD
|
||||
COMMAND "${CMAKE_COMMAND}" -E copy "${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" $<TARGET_FILE_DIR:mimalloc>
|
||||
COMMENT "Copy mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll to output directory")
|
||||
install(FILES "$<TARGET_FILE_DIR:mimalloc>/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_libdir})
|
||||
install(FILES "$<TARGET_FILE_DIR:mimalloc>/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.dll" DESTINATION ${mi_install_bindir})
|
||||
endif()
|
||||
|
||||
install(TARGETS mimalloc EXPORT mimalloc DESTINATION ${mi_install_libdir} LIBRARY)
|
||||
install(TARGETS mimalloc EXPORT mimalloc ARCHIVE DESTINATION ${mi_install_libdir} RUNTIME DESTINATION ${mi_install_bindir} LIBRARY DESTINATION ${mi_install_libdir})
|
||||
install(EXPORT mimalloc DESTINATION ${mi_install_cmakedir})
|
||||
endif()
|
||||
|
||||
@ -522,6 +540,15 @@ if (MI_BUILD_OBJECT)
|
||||
endif()
|
||||
|
||||
# pkg-config file support
|
||||
set(pc_libraries "")
|
||||
foreach(item IN LISTS mi_libraries)
|
||||
if(item MATCHES " *[-].*")
|
||||
set(pc_libraries "${pc_libraries} ${item}")
|
||||
else()
|
||||
set(pc_libraries "${pc_libraries} -l${item}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
include("cmake/JoinPaths.cmake")
|
||||
join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||
join_paths(libdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_LIBDIR}")
|
||||
@ -530,6 +557,8 @@ configure_file(mimalloc.pc.in mimalloc.pc @ONLY)
|
||||
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/mimalloc.pc"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
|
||||
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# API surface testing
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
set(mi_version_major 2)
|
||||
set(mi_version_minor 1)
|
||||
set(mi_version_patch 6)
|
||||
set(mi_version_patch 7)
|
||||
set(mi_version ${mi_version_major}.${mi_version_minor})
|
||||
|
||||
set(PACKAGE_VERSION ${mi_version})
|
||||
|
||||
28
docker/alpine-arm32v7/Dockerfile
Normal file
28
docker/alpine-arm32v7/Dockerfile
Normal file
@ -0,0 +1,28 @@
|
||||
# install from an image
|
||||
# download first an appropiate tar.gz image into the current directory
|
||||
# from: <https://github.com/alpinelinux/docker-alpine/tree/edge/armv7>
|
||||
FROM scratch
|
||||
|
||||
# Substitute the image name that was downloaded
|
||||
ADD alpine-minirootfs-20240329-armv7.tar.gz /
|
||||
|
||||
# Install tools
|
||||
RUN apk add build-base make cmake
|
||||
RUN apk add git
|
||||
RUN apk add vim
|
||||
|
||||
RUN mkdir -p /home/dev
|
||||
WORKDIR /home/dev
|
||||
|
||||
# Get mimalloc
|
||||
RUN git clone https://github.com/microsoft/mimalloc -b dev-slice
|
||||
RUN mkdir -p mimalloc/out/release
|
||||
RUN mkdir -p mimalloc/out/debug
|
||||
|
||||
# Build mimalloc debug
|
||||
WORKDIR /home/dev/mimalloc/out/debug
|
||||
RUN cmake ../.. -DMI_DEBUG_FULL=ON
|
||||
RUN make -j
|
||||
RUN make test
|
||||
|
||||
CMD ["/bin/sh"]
|
||||
23
docker/alpine/Dockerfile
Normal file
23
docker/alpine/Dockerfile
Normal file
@ -0,0 +1,23 @@
|
||||
# alpine image
|
||||
FROM alpine
|
||||
|
||||
# Install tools
|
||||
RUN apk add build-base make cmake
|
||||
RUN apk add git
|
||||
RUN apk add vim
|
||||
|
||||
RUN mkdir -p /home/dev
|
||||
WORKDIR /home/dev
|
||||
|
||||
# Get mimalloc
|
||||
RUN git clone https://github.com/microsoft/mimalloc -b dev-slice
|
||||
RUN mkdir -p mimalloc/out/release
|
||||
RUN mkdir -p mimalloc/out/debug
|
||||
|
||||
# Build mimalloc debug
|
||||
WORKDIR /home/dev/mimalloc/out/debug
|
||||
RUN cmake ../.. -DMI_DEBUG_FULL=ON
|
||||
RUN make -j
|
||||
RUN make test
|
||||
|
||||
CMD ["/bin/sh"]
|
||||
23
docker/manylinux-x64/Dockerfile
Normal file
23
docker/manylinux-x64/Dockerfile
Normal file
@ -0,0 +1,23 @@
|
||||
FROM quay.io/pypa/manylinux2014_x86_64
|
||||
|
||||
# Install tools
|
||||
RUN yum install -y openssl-devel
|
||||
RUN yum install -y gcc gcc-c++ kernel-devel make
|
||||
RUN yum install -y git cmake
|
||||
RUN yum install -y vim
|
||||
|
||||
RUN mkdir -p /home/dev
|
||||
WORKDIR /home/dev
|
||||
|
||||
# Get mimalloc
|
||||
RUN git clone https://github.com/microsoft/mimalloc -b dev-slice
|
||||
RUN mkdir -p mimalloc/out/release
|
||||
RUN mkdir -p mimalloc/out/debug
|
||||
|
||||
# Build mimalloc debug
|
||||
WORKDIR /home/dev/mimalloc/out/debug
|
||||
RUN cmake ../.. -DMI_DEBUG_FULL=ON
|
||||
RUN make -j
|
||||
RUN make test
|
||||
|
||||
CMD ["/bin/sh"]
|
||||
10
docker/readme.md
Normal file
10
docker/readme.md
Normal file
@ -0,0 +1,10 @@
|
||||
Various example docker files used for testing.
|
||||
|
||||
Usage:
|
||||
|
||||
```
|
||||
> cd <host>
|
||||
> docker build -t <host>-mimalloc .
|
||||
> docker run -it <host>-mimalloc
|
||||
>> make test
|
||||
```
|
||||
@ -43,6 +43,7 @@ not accidentally mix pointers from different allocators).
|
||||
#define reallocf(p,n) mi_reallocf(p,n)
|
||||
#define malloc_size(p) mi_usable_size(p)
|
||||
#define malloc_usable_size(p) mi_usable_size(p)
|
||||
#define malloc_good_size(sz) mi_malloc_good_size(sz)
|
||||
#define cfree(p) mi_free(p)
|
||||
|
||||
#define valloc(n) mi_valloc(n)
|
||||
|
||||
@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
#ifndef MIMALLOC_H
|
||||
#define MIMALLOC_H
|
||||
|
||||
#define MI_MALLOC_VERSION 216 // major + 2 digits minor
|
||||
#define MI_MALLOC_VERSION 217 // major + 2 digits minor
|
||||
|
||||
// ------------------------------------------------------
|
||||
// Compiler specific attributes
|
||||
@ -328,7 +328,7 @@ typedef enum mi_option_e {
|
||||
mi_option_allow_large_os_pages, // allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process.
|
||||
mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup
|
||||
mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node
|
||||
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup
|
||||
mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`)
|
||||
mi_option_deprecated_segment_cache,
|
||||
mi_option_deprecated_page_reset,
|
||||
mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination
|
||||
@ -342,11 +342,12 @@ typedef enum mi_option_e {
|
||||
mi_option_max_warnings, // issue at most N warning messages
|
||||
mi_option_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%)
|
||||
mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
|
||||
mi_option_arena_reserve, // initial memory size in KiB for arena reservation (= 1 GiB on 64-bit)
|
||||
mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`)
|
||||
mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10)
|
||||
mi_option_purge_extend_delay,
|
||||
mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1)
|
||||
mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's)
|
||||
mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
|
||||
_mi_option_last,
|
||||
// legacy option names
|
||||
mi_option_large_os_pages = mi_option_allow_large_os_pages,
|
||||
|
||||
@ -132,7 +132,7 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
|
||||
|
||||
#elif defined(_MSC_VER)
|
||||
|
||||
// MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics.
|
||||
// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
@ -201,7 +201,7 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_
|
||||
#else
|
||||
uintptr_t x = *p;
|
||||
if (mo > mi_memory_order_relaxed) {
|
||||
while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
|
||||
while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
|
||||
}
|
||||
return x;
|
||||
#endif
|
||||
|
||||
@ -14,8 +14,8 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
// functions and macros.
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
#include "mimalloc/types.h"
|
||||
#include "mimalloc/track.h"
|
||||
#include "types.h"
|
||||
#include "track.h"
|
||||
|
||||
#if (MI_DEBUG>0)
|
||||
#define mi_trace_message(...) _mi_trace_message(__VA_ARGS__)
|
||||
@ -88,6 +88,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
|
||||
mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap
|
||||
void _mi_thread_done(mi_heap_t* heap);
|
||||
void _mi_thread_data_collect(void);
|
||||
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap);
|
||||
|
||||
// os.c
|
||||
void _mi_os_init(void); // called from process init
|
||||
@ -186,11 +187,13 @@ size_t _mi_bin_size(uint8_t bin); // for stats
|
||||
uint8_t _mi_bin(size_t size); // for stats
|
||||
|
||||
// "heap.c"
|
||||
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag);
|
||||
void _mi_heap_destroy_pages(mi_heap_t* heap);
|
||||
void _mi_heap_collect_abandon(mi_heap_t* heap);
|
||||
void _mi_heap_set_default_direct(mi_heap_t* heap);
|
||||
bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid);
|
||||
void _mi_heap_unsafe_destroy_all(void);
|
||||
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag);
|
||||
|
||||
// "stats.c"
|
||||
void _mi_stats_done(mi_stats_t* stats);
|
||||
@ -379,10 +382,10 @@ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
|
||||
}
|
||||
#else /* __builtin_umul_overflow is unavailable */
|
||||
static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) {
|
||||
#define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
|
||||
#define MI_MUL_COULD_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX)
|
||||
*total = count * size;
|
||||
// note: gcc/clang optimize this to directly check the overflow flag
|
||||
return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count);
|
||||
return ((size >= MI_MUL_COULD_OVERFLOW || count >= MI_MUL_COULD_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count);
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -546,6 +549,7 @@ static inline mi_heap_t* mi_page_heap(const mi_page_t* page) {
|
||||
static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) {
|
||||
mi_assert_internal(mi_page_thread_free_flag(page) != MI_DELAYED_FREEING);
|
||||
mi_atomic_store_release(&page->xheap,(uintptr_t)heap);
|
||||
if (heap != NULL) { page->heap_tag = heap->tag; }
|
||||
}
|
||||
|
||||
// Thread free flag helpers
|
||||
|
||||
@ -26,7 +26,7 @@ typedef struct mi_os_mem_config_s {
|
||||
size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows)
|
||||
size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB)
|
||||
bool has_overcommit; // can we reserve more memory than can be actually committed?
|
||||
bool must_free_whole; // must allocated blocks be freed as a whole (false for mmap, true for VirtualAlloc)
|
||||
bool has_partial_free; // can allocated blocks be freed partially? (true for mmap, false for VirtualAlloc)
|
||||
bool has_virtual_reserve; // supports virtual address space reservation? (if true we can reserve virtual address space without using commit or physical memory)
|
||||
} mi_os_mem_config_t;
|
||||
|
||||
@ -208,13 +208,18 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce
|
||||
// but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
|
||||
// Nevertheless, it seems needed on older graviton platforms (see issue #851).
|
||||
// For now, we only enable this for specific platforms.
|
||||
#if defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__) /* special case aarch64 for older gcc versions (issue #851) */ \
|
||||
&& !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
|
||||
#if !defined(__APPLE__) /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
|
||||
&& !defined(MI_LIBC_MUSL) \
|
||||
&& (!defined(__clang_major__) || __clang_major__ >= 14) /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
|
||||
#define MI_USE_BUILTIN_THREAD_POINTER 1
|
||||
#if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
|
||||
|| (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
|
||||
|| (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
|
||||
#define MI_USE_BUILTIN_THREAD_POINTER 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
// defined in `init.c`; do not use these directly
|
||||
extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from
|
||||
extern bool _mi_process_is_initialized; // has mi_process_init been called?
|
||||
@ -222,7 +227,13 @@ extern bool _mi_process_is_initialized; // has mi_process_init been
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept;
|
||||
|
||||
// Get a unique id for the current thread.
|
||||
#if defined(_WIN32)
|
||||
#if defined(MI_PRIM_THREAD_ID)
|
||||
|
||||
static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
|
||||
return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488)
|
||||
}
|
||||
|
||||
#elif defined(_WIN32)
|
||||
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
|
||||
@ -24,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
|
||||
#include <stddef.h> // ptrdiff_t
|
||||
#include <stdint.h> // uintptr_t, uint16_t, etc
|
||||
#include "mimalloc/atomic.h" // _Atomic
|
||||
#include "atomic.h" // _Atomic
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(disable:4214) // bitfield is not int
|
||||
@ -319,6 +319,7 @@ typedef struct mi_page_s {
|
||||
mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`)
|
||||
uint16_t used; // number of blocks in use (including blocks in `thread_free`)
|
||||
uint8_t block_size_shift; // if not zero, then `(1 << block_size_shift) == block_size` (only used for fast path in `free.c:_mi_page_ptr_unalign`)
|
||||
uint8_t heap_tag; // tag of the owning heap, used for separated heaps by object type
|
||||
// padding
|
||||
size_t block_size; // size available in each block (always `>0`)
|
||||
uint8_t* page_start; // start of the page area containing the blocks
|
||||
@ -538,6 +539,7 @@ struct mi_heap_s {
|
||||
size_t page_retired_max; // largest retired index into the `pages` array.
|
||||
mi_heap_t* next; // list of heaps per thread
|
||||
bool no_reclaim; // `true` if this heap should not reclaim abandoned pages
|
||||
uint8_t tag; // custom tag, can be used for separating heaps based on the object types
|
||||
mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size.
|
||||
mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin")
|
||||
};
|
||||
|
||||
@ -12,8 +12,8 @@ is a general purpose allocator with excellent [performance](#performance) charac
|
||||
Initially developed by Daan Leijen for the runtime systems of the
|
||||
[Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages.
|
||||
|
||||
Latest release tag: `v2.1.6` (2024-05-13).
|
||||
Latest v1 tag: `v1.8.6` (2024-05-13).
|
||||
Latest release tag: `v2.1.7` (2024-05-21).
|
||||
Latest v1 tag: `v1.8.7` (2024-05-21).
|
||||
|
||||
mimalloc is a drop-in replacement for `malloc` and can be used in other programs
|
||||
without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as:
|
||||
@ -82,6 +82,8 @@ memory usage
|
||||
and fragmentation compared to mimalloc `v1.x` (especially for large workloads). Should otherwise have similar performance
|
||||
(see [below](#performance)); please report if you observe any significant performance regression.
|
||||
|
||||
* 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches
|
||||
from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches.
|
||||
* 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation.
|
||||
* 2024-04-22, `v1.8.4`, `v2.1.4`: Fixes various bugs and build issues. Add `MI_LIBC_MUSL` cmake flag for musl builds.
|
||||
Free-ing code is refactored into a separate module (`free.c`). Mimalloc page info is simplified with the block size
|
||||
|
||||
14
src/arena.c
14
src/arena.c
@ -57,6 +57,7 @@ typedef struct mi_arena_s {
|
||||
mi_bitmap_field_t* blocks_purge; // blocks that can be (reset) decommitted. (can be NULL for memory that cannot be (reset) decommitted)
|
||||
mi_bitmap_field_t* blocks_abandoned; // blocks that start with an abandoned segment. (This crosses API's but it is convenient to have here)
|
||||
mi_bitmap_field_t blocks_inuse[1]; // in-place bitmap of in-use blocks (of size `field_count`)
|
||||
// do not add further fields here as the dirty, committed, purged, and abandoned bitmaps follow the inuse bitmap fields.
|
||||
} mi_arena_t;
|
||||
|
||||
|
||||
@ -144,18 +145,19 @@ static bool mi_arena_memid_indices(mi_memid_t memid, size_t* arena_index, mi_bit
|
||||
|
||||
#define MI_ARENA_STATIC_MAX (MI_INTPTR_SIZE*MI_KiB) // 8 KiB on 64-bit
|
||||
|
||||
static uint8_t mi_arena_static[MI_ARENA_STATIC_MAX];
|
||||
static _Atomic(size_t) mi_arena_static_top;
|
||||
static mi_decl_cache_align uint8_t mi_arena_static[MI_ARENA_STATIC_MAX]; // must be cache aligned, see issue #895
|
||||
static mi_decl_cache_align _Atomic(size_t) mi_arena_static_top;
|
||||
|
||||
static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* memid) {
|
||||
*memid = _mi_memid_none();
|
||||
if (size == 0 || size > MI_ARENA_STATIC_MAX) return NULL;
|
||||
if ((mi_atomic_load_relaxed(&mi_arena_static_top) + size) > MI_ARENA_STATIC_MAX) return NULL;
|
||||
const size_t toplow = mi_atomic_load_relaxed(&mi_arena_static_top);
|
||||
if ((toplow + size) > MI_ARENA_STATIC_MAX) return NULL;
|
||||
|
||||
// try to claim space
|
||||
if (alignment == 0) { alignment = 1; }
|
||||
if (alignment < MI_MAX_ALIGN_SIZE) { alignment = MI_MAX_ALIGN_SIZE; }
|
||||
const size_t oversize = size + alignment - 1;
|
||||
if (oversize > MI_ARENA_STATIC_MAX) return NULL;
|
||||
if (toplow + oversize > MI_ARENA_STATIC_MAX) return NULL;
|
||||
const size_t oldtop = mi_atomic_add_acq_rel(&mi_arena_static_top, oversize);
|
||||
size_t top = oldtop + oversize;
|
||||
if (top > MI_ARENA_STATIC_MAX) {
|
||||
@ -169,7 +171,7 @@ static void* mi_arena_static_zalloc(size_t size, size_t alignment, mi_memid_t* m
|
||||
memid->initially_zero = true;
|
||||
const size_t start = _mi_align_up(oldtop, alignment);
|
||||
uint8_t* const p = &mi_arena_static[start];
|
||||
_mi_memzero(p, size);
|
||||
_mi_memzero_aligned(p, size);
|
||||
return p;
|
||||
}
|
||||
|
||||
|
||||
54
src/heap.c
54
src/heap.c
@ -128,6 +128,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
|
||||
const bool force = (collect >= MI_FORCE);
|
||||
_mi_deferred_free(heap, force);
|
||||
|
||||
// python/cpython#112532: we may be called from a thread that is not the owner of the heap
|
||||
const bool is_main_thread = (_mi_is_main_thread() && heap->thread_id == _mi_thread_id());
|
||||
|
||||
// note: never reclaim on collect but leave it to threads that need storage to reclaim
|
||||
const bool force_main =
|
||||
#ifdef NDEBUG
|
||||
@ -135,7 +138,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
|
||||
#else
|
||||
collect >= MI_FORCE
|
||||
#endif
|
||||
&& _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim;
|
||||
&& is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim;
|
||||
|
||||
if (force_main) {
|
||||
// the main thread is abandoned (end-of-program), try to reclaim all abandoned segments.
|
||||
@ -164,7 +167,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect)
|
||||
_mi_abandoned_collect(heap, collect == MI_FORCE /* force? */, &heap->tld->segments);
|
||||
|
||||
// if forced, collect thread data cache on program-exit (or shared library unload)
|
||||
if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) {
|
||||
if (force && is_main_thread && mi_heap_is_backing(heap)) {
|
||||
_mi_thread_data_collect(); // collect thread data cache
|
||||
}
|
||||
|
||||
@ -208,22 +211,33 @@ mi_heap_t* mi_heap_get_backing(void) {
|
||||
return bheap;
|
||||
}
|
||||
|
||||
void _mi_heap_init(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool noreclaim, uint8_t tag) {
|
||||
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
|
||||
heap->tld = tld;
|
||||
heap->thread_id = _mi_thread_id();
|
||||
heap->arena_id = arena_id;
|
||||
heap->no_reclaim = noreclaim;
|
||||
heap->tag = tag;
|
||||
if (heap == tld->heap_backing) {
|
||||
_mi_random_init(&heap->random);
|
||||
}
|
||||
else {
|
||||
_mi_random_split(&tld->heap_backing->random, &heap->random);
|
||||
}
|
||||
heap->cookie = _mi_heap_random_next(heap) | 1;
|
||||
heap->keys[0] = _mi_heap_random_next(heap);
|
||||
heap->keys[1] = _mi_heap_random_next(heap);
|
||||
// push on the thread local heaps list
|
||||
heap->next = heap->tld->heaps;
|
||||
heap->tld->heaps = heap;
|
||||
}
|
||||
|
||||
mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) {
|
||||
mi_heap_t* bheap = mi_heap_get_backing();
|
||||
mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode?
|
||||
if (heap == NULL) return NULL;
|
||||
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t));
|
||||
heap->tld = bheap->tld;
|
||||
heap->thread_id = _mi_thread_id();
|
||||
heap->arena_id = arena_id;
|
||||
_mi_random_split(&bheap->random, &heap->random);
|
||||
heap->cookie = _mi_heap_random_next(heap) | 1;
|
||||
heap->keys[0] = _mi_heap_random_next(heap);
|
||||
heap->keys[1] = _mi_heap_random_next(heap);
|
||||
heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe
|
||||
// push on the thread local heaps list
|
||||
heap->next = heap->tld->heaps;
|
||||
heap->tld->heaps = heap;
|
||||
// don't reclaim abandoned pages or otherwise destroy is unsafe
|
||||
_mi_heap_init(heap, bheap->tld, arena_id, true /* no reclaim */, 0 /* default tag */);
|
||||
return heap;
|
||||
}
|
||||
|
||||
@ -281,6 +295,18 @@ static void mi_heap_free(mi_heap_t* heap) {
|
||||
mi_free(heap);
|
||||
}
|
||||
|
||||
// return a heap on the same thread as `heap` specialized for the specified tag (if it exists)
|
||||
mi_heap_t* _mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) {
|
||||
if (heap->tag == tag) {
|
||||
return heap;
|
||||
}
|
||||
for (mi_heap_t *curr = heap->tld->heaps; curr != NULL; curr = curr->next) {
|
||||
if (curr->tag == tag) {
|
||||
return curr;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Heap destroy
|
||||
|
||||
40
src/init.c
40
src/init.c
@ -25,6 +25,7 @@ const mi_page_t _mi_page_empty = {
|
||||
NULL, // local_free
|
||||
0, // used
|
||||
0, // block size shift
|
||||
0, // heap tag
|
||||
0, // block_size
|
||||
NULL, // page_start
|
||||
#if (MI_PADDING || MI_ENCODE_FREELIST)
|
||||
@ -33,9 +34,7 @@ const mi_page_t _mi_page_empty = {
|
||||
MI_ATOMIC_VAR_INIT(0), // xthread_free
|
||||
MI_ATOMIC_VAR_INIT(0), // xheap
|
||||
NULL, NULL
|
||||
#if MI_INTPTR_SIZE==8
|
||||
, { 0 } // padding
|
||||
#endif
|
||||
};
|
||||
|
||||
#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty)
|
||||
@ -124,7 +123,8 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
|
||||
0, // page count
|
||||
MI_BIN_FULL, 0, // page retired min/max
|
||||
NULL, // next
|
||||
false,
|
||||
false, // can reclaim
|
||||
0, // tag
|
||||
MI_SMALL_PAGES_EMPTY,
|
||||
MI_PAGE_QUEUES_EMPTY
|
||||
};
|
||||
@ -170,6 +170,7 @@ mi_heap_t _mi_heap_main = {
|
||||
MI_BIN_FULL, 0, // page retired min/max
|
||||
NULL, // next heap
|
||||
false, // can reclaim
|
||||
0, // tag
|
||||
MI_SMALL_PAGES_EMPTY,
|
||||
MI_PAGE_QUEUES_EMPTY
|
||||
};
|
||||
@ -288,7 +289,7 @@ void _mi_thread_data_collect(void) {
|
||||
}
|
||||
|
||||
// Initialize the thread local default heap, called from `mi_thread_init`
|
||||
static bool _mi_heap_init(void) {
|
||||
static bool _mi_thread_heap_init(void) {
|
||||
if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true;
|
||||
if (_mi_is_main_thread()) {
|
||||
// mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization
|
||||
@ -304,26 +305,25 @@ static bool _mi_heap_init(void) {
|
||||
|
||||
mi_tld_t* tld = &td->tld;
|
||||
mi_heap_t* heap = &td->heap;
|
||||
_mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld));
|
||||
_mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap));
|
||||
heap->thread_id = _mi_thread_id();
|
||||
_mi_random_init(&heap->random);
|
||||
heap->cookie = _mi_heap_random_next(heap) | 1;
|
||||
heap->keys[0] = _mi_heap_random_next(heap);
|
||||
heap->keys[1] = _mi_heap_random_next(heap);
|
||||
heap->tld = tld;
|
||||
tld->heap_backing = heap;
|
||||
tld->heaps = heap;
|
||||
tld->segments.stats = &tld->stats;
|
||||
tld->segments.os = &tld->os;
|
||||
tld->os.stats = &tld->stats;
|
||||
_mi_tld_init(tld, heap); // must be before `_mi_heap_init`
|
||||
_mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */);
|
||||
_mi_heap_set_default_direct(heap);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// initialize thread local data
|
||||
void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) {
|
||||
_mi_memcpy_aligned(tld, &tld_empty, sizeof(mi_tld_t));
|
||||
tld->heap_backing = bheap;
|
||||
tld->heaps = NULL;
|
||||
tld->segments.stats = &tld->stats;
|
||||
tld->segments.os = &tld->os;
|
||||
tld->os.stats = &tld->stats;
|
||||
}
|
||||
|
||||
// Free the thread local default heap (called from `mi_thread_done`)
|
||||
static bool _mi_heap_done(mi_heap_t* heap) {
|
||||
static bool _mi_thread_heap_done(mi_heap_t* heap) {
|
||||
if (!mi_heap_is_initialized(heap)) return true;
|
||||
|
||||
// reset default heap
|
||||
@ -420,7 +420,7 @@ void mi_thread_init(void) mi_attr_noexcept
|
||||
// initialize the thread local default heap
|
||||
// (this will call `_mi_heap_set_default_direct` and thus set the
|
||||
// fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called)
|
||||
if (_mi_heap_init()) return; // returns true if already initialized
|
||||
if (_mi_thread_heap_init()) return; // returns true if already initialized
|
||||
|
||||
_mi_stat_increase(&_mi_stats_main.threads, 1);
|
||||
mi_atomic_increment_relaxed(&thread_count);
|
||||
@ -452,7 +452,7 @@ void _mi_thread_done(mi_heap_t* heap)
|
||||
if (heap->thread_id != _mi_thread_id()) return;
|
||||
|
||||
// abandon the thread local heap
|
||||
if (_mi_heap_done(heap)) return; // returns true if already ran
|
||||
if (_mi_thread_heap_done(heap)) return; // returns true if already ran
|
||||
}
|
||||
|
||||
void _mi_heap_set_default_direct(mi_heap_t* heap) {
|
||||
|
||||
@ -65,7 +65,7 @@ static mi_option_desc_t options[_mi_option_last] =
|
||||
{ 0, UNINIT, MI_OPTION_LEGACY(allow_large_os_pages,large_os_pages) }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's
|
||||
{ 0, UNINIT, MI_OPTION(reserve_huge_os_pages) }, // per 1GiB huge pages
|
||||
{-1, UNINIT, MI_OPTION(reserve_huge_os_pages_at) }, // reserve huge pages at node N
|
||||
{ 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve OS memory in advance
|
||||
{ 0, UNINIT, MI_OPTION(reserve_os_memory) }, // reserve N KiB OS memory in advance (use `option_get_size`)
|
||||
{ 0, UNINIT, MI_OPTION(deprecated_segment_cache) }, // cache N segments per thread
|
||||
{ 0, UNINIT, MI_OPTION(deprecated_page_reset) }, // reset page memory on free
|
||||
{ 0, UNINIT, MI_OPTION_LEGACY(abandoned_page_purge,abandoned_page_reset) }, // reset free page memory when a thread terminates
|
||||
@ -79,19 +79,20 @@ static mi_option_desc_t options[_mi_option_last] =
|
||||
{ 0, UNINIT, MI_OPTION(use_numa_nodes) }, // 0 = use available numa nodes, otherwise use at most N nodes.
|
||||
{ 0, UNINIT, MI_OPTION_LEGACY(disallow_os_alloc,limit_os_alloc) }, // 1 = do not use OS memory for allocation (but only reserved arenas)
|
||||
{ 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose
|
||||
{ 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output
|
||||
{ 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
|
||||
{ 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments per try.
|
||||
{ 32, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output
|
||||
{ 32, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output
|
||||
{ 10, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try.
|
||||
{ 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees!
|
||||
#if (MI_INTPTR_SIZE>4)
|
||||
{ 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time
|
||||
{ 1024L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`)
|
||||
#else
|
||||
{ 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) },
|
||||
{ 128L*1024L, UNINIT, MI_OPTION(arena_reserve) }, // =128MiB on 32-bit
|
||||
#endif
|
||||
{ 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's
|
||||
{ 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) },
|
||||
{ 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) },// reclaim an abandoned segment on a free
|
||||
{ 0, UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's)
|
||||
{ 400, UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries.
|
||||
};
|
||||
|
||||
static void mi_option_init(mi_option_desc_t* desc);
|
||||
@ -135,8 +136,12 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma
|
||||
|
||||
mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) {
|
||||
mi_assert_internal(mi_option_has_size_in_kib(option));
|
||||
long x = mi_option_get(option);
|
||||
return (x < 0 ? 0 : (size_t)x * MI_KiB);
|
||||
const long x = mi_option_get(option);
|
||||
size_t size = (x < 0 ? 0 : (size_t)x);
|
||||
if (mi_option_has_size_in_kib(option)) {
|
||||
size *= MI_KiB;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
void mi_option_set(mi_option_t option, long value) {
|
||||
@ -479,14 +484,20 @@ static void mi_option_init(mi_option_desc_t* desc) {
|
||||
else {
|
||||
char* end = buf;
|
||||
long value = strtol(buf, &end, 10);
|
||||
if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_arena_reserve) {
|
||||
// this option is interpreted in KiB to prevent overflow of `long`
|
||||
if (mi_option_has_size_in_kib(desc->option)) {
|
||||
// this option is interpreted in KiB to prevent overflow of `long` for large allocations
|
||||
// (long is 32-bit on 64-bit windows, which allows for 4TiB max.)
|
||||
size_t size = (value < 0 ? 0 : (size_t)value);
|
||||
bool overflow = false;
|
||||
if (*end == 'K') { end++; }
|
||||
else if (*end == 'M') { value *= MI_KiB; end++; }
|
||||
else if (*end == 'G') { value *= MI_MiB; end++; }
|
||||
else { value = (value + MI_KiB - 1) / MI_KiB; }
|
||||
if (end[0] == 'I' && end[1] == 'B') { end += 2; }
|
||||
else if (*end == 'B') { end++; }
|
||||
else if (*end == 'M') { overflow = mi_mul_overflow(size,MI_KiB,&size); end++; }
|
||||
else if (*end == 'G') { overflow = mi_mul_overflow(size,MI_MiB,&size); end++; }
|
||||
else if (*end == 'T') { overflow = mi_mul_overflow(size,MI_GiB,&size); end++; }
|
||||
else { size = (size + MI_KiB - 1) / MI_KiB; }
|
||||
if (end[0] == 'I' && end[1] == 'B') { end += 2; } // KiB, MiB, GiB, TiB
|
||||
else if (*end == 'B') { end++; } // Kb, Mb, Gb, Tb
|
||||
if (overflow || size > MI_MAX_ALLOC_SIZE) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); }
|
||||
value = (size > LONG_MAX ? LONG_MAX : (long)size);
|
||||
}
|
||||
if (*end == 0) {
|
||||
desc->value = value;
|
||||
|
||||
10
src/os.c
10
src/os.c
@ -12,8 +12,6 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
|
||||
/* -----------------------------------------------------------
|
||||
Initialization.
|
||||
On windows initializes support for aligned allocation and
|
||||
large OS pages (if MIMALLOC_LARGE_OS_PAGES is true).
|
||||
----------------------------------------------------------- */
|
||||
|
||||
static mi_os_mem_config_t mi_os_mem_config = {
|
||||
@ -21,7 +19,7 @@ static mi_os_mem_config_t mi_os_mem_config = {
|
||||
0, // large page size (usually 2MiB)
|
||||
4096, // allocation granularity
|
||||
true, // has overcommit? (if true we use MAP_NORESERVE on mmap systems)
|
||||
false, // must free whole? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
|
||||
false, // can we partially free allocated blocks? (on mmap systems we can free anywhere in a mapped range, but on Windows we must free the entire span)
|
||||
true // has virtual reserve? (if true we can reserve virtual address space without using commit or physical memory)
|
||||
};
|
||||
|
||||
@ -239,7 +237,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
|
||||
if (size >= (SIZE_MAX - alignment)) return NULL; // overflow
|
||||
const size_t over_size = size + alignment;
|
||||
|
||||
if (mi_os_mem_config.must_free_whole) { // win32 virtualAlloc cannot free parts of an allocate block
|
||||
if (!mi_os_mem_config.has_partial_free) { // win32 virtualAlloc cannot free parts of an allocated block
|
||||
// over-allocate uncommitted (virtual) memory
|
||||
p = mi_os_prim_alloc(over_size, 1 /*alignment*/, false /* commit? */, false /* allow_large */, is_large, is_zero, stats);
|
||||
if (p == NULL) return NULL;
|
||||
@ -260,7 +258,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
|
||||
p = mi_os_prim_alloc(over_size, 1, commit, false, is_large, is_zero, stats);
|
||||
if (p == NULL) return NULL;
|
||||
|
||||
// and selectively unmap parts around the over-allocated area. (noop on sbrk)
|
||||
// and selectively unmap parts around the over-allocated area.
|
||||
void* aligned_p = mi_align_up_ptr(p, alignment);
|
||||
size_t pre_size = (uint8_t*)aligned_p - (uint8_t*)p;
|
||||
size_t mid_size = _mi_align_up(size, _mi_os_page_size());
|
||||
@ -268,7 +266,7 @@ static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit
|
||||
mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size);
|
||||
if (pre_size > 0) { mi_os_prim_free(p, pre_size, commit, stats); }
|
||||
if (post_size > 0) { mi_os_prim_free((uint8_t*)aligned_p + mid_size, post_size, commit, stats); }
|
||||
// we can return the aligned pointer on `mmap` (and sbrk) systems
|
||||
// we can return the aligned pointer on `mmap` systems
|
||||
p = aligned_p;
|
||||
*base = aligned_p; // since we freed the pre part, `*base == p`.
|
||||
}
|
||||
|
||||
@ -51,7 +51,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) {
|
||||
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
|
||||
config->alloc_granularity = 16;
|
||||
config->has_overcommit = false;
|
||||
config->must_free_whole = true;
|
||||
config->has_partial_free = false;
|
||||
config->has_virtual_reserve = false;
|
||||
}
|
||||
|
||||
|
||||
@ -422,6 +422,7 @@ __attribute__((constructor(0)))
|
||||
#else
|
||||
__attribute__((constructor)) // seems not supported by g++-11 on the M1
|
||||
#endif
|
||||
__attribute__((used))
|
||||
static void _mi_macos_override_malloc(void) {
|
||||
malloc_zone_t* purgeable_zone = NULL;
|
||||
|
||||
|
||||
@ -57,7 +57,7 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
#include <sys/sysctl.h>
|
||||
#endif
|
||||
|
||||
#if !defined(__HAIKU__) && !defined(__APPLE__) && !defined(__CYGWIN__) && !defined(__OpenBSD__) && !defined(__sun)
|
||||
#if defined(__linux__) || defined(__FreeBSD__)
|
||||
#define MI_HAS_SYSCALL_H
|
||||
#include <sys/syscall.h>
|
||||
#endif
|
||||
@ -67,37 +67,36 @@ terms of the MIT license. A copy of the license can be found in the file
|
||||
// Use syscalls for some primitives to allow for libraries that override open/read/close etc.
|
||||
// and do allocation themselves; using syscalls prevents recursion when mimalloc is
|
||||
// still initializing (issue #713)
|
||||
// Declare inline to avoid unused function warnings.
|
||||
//------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access)
|
||||
|
||||
static int mi_prim_open(const char* fpath, int open_flags) {
|
||||
static inline int mi_prim_open(const char* fpath, int open_flags) {
|
||||
return syscall(SYS_open,fpath,open_flags,0);
|
||||
}
|
||||
static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
|
||||
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
|
||||
return syscall(SYS_read,fd,buf,bufsize);
|
||||
}
|
||||
static int mi_prim_close(int fd) {
|
||||
static inline int mi_prim_close(int fd) {
|
||||
return syscall(SYS_close,fd);
|
||||
}
|
||||
static int mi_prim_access(const char *fpath, int mode) {
|
||||
static inline int mi_prim_access(const char *fpath, int mode) {
|
||||
return syscall(SYS_access,fpath,mode);
|
||||
}
|
||||
|
||||
#elif !defined(__sun) && \
|
||||
(!defined(__APPLE__) || (MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_7)) // avoid unused warnings on macOS and Solaris
|
||||
#else
|
||||
|
||||
static int mi_prim_open(const char* fpath, int open_flags) {
|
||||
static inline int mi_prim_open(const char* fpath, int open_flags) {
|
||||
return open(fpath,open_flags);
|
||||
}
|
||||
static ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
|
||||
static inline ssize_t mi_prim_read(int fd, void* buf, size_t bufsize) {
|
||||
return read(fd,buf,bufsize);
|
||||
}
|
||||
static int mi_prim_close(int fd) {
|
||||
static inline int mi_prim_close(int fd) {
|
||||
return close(fd);
|
||||
}
|
||||
static int mi_prim_access(const char *fpath, int mode) {
|
||||
static inline int mi_prim_access(const char *fpath, int mode) {
|
||||
return access(fpath,mode);
|
||||
}
|
||||
|
||||
@ -144,7 +143,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config )
|
||||
}
|
||||
config->large_page_size = 2*MI_MiB; // TODO: can we query the OS for this?
|
||||
config->has_overcommit = unix_detect_overcommit();
|
||||
config->must_free_whole = false; // mmap can free in parts
|
||||
config->has_partial_free = true; // mmap can free in parts
|
||||
config->has_virtual_reserve = true; // todo: check if this true for NetBSD? (for anonymous mmap with PROT_NONE)
|
||||
|
||||
// disable transparent huge pages for this process?
|
||||
@ -199,7 +198,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
|
||||
p = mmap(addr, size, protect_flags, flags | MAP_ALIGNED(n), fd, 0);
|
||||
if (p==MAP_FAILED || !_mi_is_aligned(p,try_alignment)) {
|
||||
int err = errno;
|
||||
_mi_warning_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
|
||||
_mi_trace_message("unable to directly request aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, addr);
|
||||
}
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
@ -224,7 +223,7 @@ static void* unix_mmap_prim(void* addr, size_t size, size_t try_alignment, int p
|
||||
#else
|
||||
int err = errno;
|
||||
#endif
|
||||
_mi_warning_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
|
||||
_mi_trace_message("unable to directly request hinted aligned OS memory (error: %d (0x%x), size: 0x%zx bytes, alignment: 0x%zx, hint address: %p)\n", err, err, size, try_alignment, hint);
|
||||
}
|
||||
if (p!=MAP_FAILED) return p;
|
||||
// fall back to regular mmap
|
||||
|
||||
@ -23,7 +23,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) {
|
||||
config->page_size = 64*MI_KiB; // WebAssembly has a fixed page size: 64KiB
|
||||
config->alloc_granularity = 16;
|
||||
config->has_overcommit = false;
|
||||
config->must_free_whole = true;
|
||||
config->has_partial_free = false;
|
||||
config->has_virtual_reserve = false;
|
||||
}
|
||||
|
||||
|
||||
@ -112,7 +112,7 @@ static bool win_enable_large_os_pages(size_t* large_page_size)
|
||||
void _mi_prim_mem_init( mi_os_mem_config_t* config )
|
||||
{
|
||||
config->has_overcommit = false;
|
||||
config->must_free_whole = true;
|
||||
config->has_partial_free = false;
|
||||
config->has_virtual_reserve = true;
|
||||
// get the page size
|
||||
SYSTEM_INFO si;
|
||||
@ -178,7 +178,7 @@ int _mi_prim_free(void* addr, size_t size ) {
|
||||
// VirtualAlloc
|
||||
//---------------------------------------------
|
||||
|
||||
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
#if (MI_INTPTR_SIZE >= 8)
|
||||
// on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations
|
||||
if (addr == NULL) {
|
||||
@ -200,13 +200,53 @@ static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignmen
|
||||
param.Arg.Pointer = &reqs;
|
||||
void* p = (*pVirtualAlloc2)(GetCurrentProcess(), addr, size, flags, PAGE_READWRITE, ¶m, 1);
|
||||
if (p != NULL) return p;
|
||||
_mi_warning_message("unable to allocate aligned OS memory (%zu bytes, error code: 0x%x, address: %p, alignment: %zu, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
|
||||
_mi_warning_message("unable to allocate aligned OS memory (0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", size, GetLastError(), addr, try_alignment, flags);
|
||||
// fall through on error
|
||||
}
|
||||
// last resort
|
||||
return VirtualAlloc(addr, size, flags, PAGE_READWRITE);
|
||||
}
|
||||
|
||||
static bool win_is_out_of_memory_error(DWORD err) {
|
||||
switch (err) {
|
||||
case ERROR_COMMITMENT_MINIMUM:
|
||||
case ERROR_COMMITMENT_LIMIT:
|
||||
case ERROR_PAGEFILE_QUOTA:
|
||||
case ERROR_NOT_ENOUGH_MEMORY:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static void* win_virtual_alloc_prim(void* addr, size_t size, size_t try_alignment, DWORD flags) {
|
||||
long max_retry_msecs = mi_option_get_clamp(mi_option_retry_on_oom, 0, 2000); // at most 2 seconds
|
||||
if (max_retry_msecs == 1) { max_retry_msecs = 100; } // if one sets the option to "true"
|
||||
for (long tries = 1; tries <= 10; tries++) { // try at most 10 times (=2200ms)
|
||||
void* p = win_virtual_alloc_prim_once(addr, size, try_alignment, flags);
|
||||
if (p != NULL) {
|
||||
// success, return the address
|
||||
return p;
|
||||
}
|
||||
else if (max_retry_msecs > 0 && (try_alignment <= 2*MI_SEGMENT_ALIGN) &&
|
||||
(flags&MEM_COMMIT) != 0 && (flags&MEM_LARGE_PAGES) == 0 &&
|
||||
win_is_out_of_memory_error(GetLastError())) {
|
||||
// if committing regular memory and being out-of-memory,
|
||||
// keep trying for a bit in case memory frees up after all. See issue #894
|
||||
_mi_warning_message("out-of-memory on OS allocation, try again... (attempt %lu, 0x%zx bytes, error code: 0x%x, address: %p, alignment: 0x%zx, flags: 0x%x)\n", tries, size, GetLastError(), addr, try_alignment, flags);
|
||||
long sleep_msecs = tries*40; // increasing waits
|
||||
if (sleep_msecs > max_retry_msecs) { sleep_msecs = max_retry_msecs; }
|
||||
max_retry_msecs -= sleep_msecs;
|
||||
Sleep(sleep_msecs);
|
||||
}
|
||||
else {
|
||||
// otherwise return with an error
|
||||
break;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void* win_virtual_alloc(void* addr, size_t size, size_t try_alignment, DWORD flags, bool large_only, bool allow_large, bool* is_large) {
|
||||
mi_assert_internal(!(large_only && !allow_large));
|
||||
static _Atomic(size_t) large_page_try_ok; // = 0;
|
||||
@ -572,6 +612,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) {
|
||||
#if !defined(MI_SHARED_LIB)
|
||||
|
||||
// use thread local storage keys to detect thread ending
|
||||
// note: another design could be to use special linker sections (see issue #869)
|
||||
#include <fibersapi.h>
|
||||
#if (_WIN32_WINNT < 0x600) // before Windows Vista
|
||||
WINBASEAPI DWORD WINAPI FlsAlloc( _In_opt_ PFLS_CALLBACK_FUNCTION lpCallback );
|
||||
|
||||
@ -347,7 +347,7 @@ uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* pa
|
||||
}
|
||||
|
||||
|
||||
static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, size_t* info_slices) {
|
||||
static size_t mi_segment_calculate_slices(size_t required, size_t* info_slices) {
|
||||
size_t page_size = _mi_os_page_size();
|
||||
size_t isize = _mi_align_up(sizeof(mi_segment_t), page_size);
|
||||
size_t guardsize = 0;
|
||||
@ -361,7 +361,6 @@ static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, siz
|
||||
}
|
||||
}
|
||||
|
||||
if (pre_size != NULL) *pre_size = isize;
|
||||
isize = _mi_align_up(isize + guardsize, MI_SEGMENT_SLICE_SIZE);
|
||||
if (info_slices != NULL) *info_slices = isize / MI_SEGMENT_SLICE_SIZE;
|
||||
size_t segment_size = (required==0 ? MI_SEGMENT_SIZE : _mi_align_up( required + isize + guardsize, MI_SEGMENT_SLICE_SIZE) );
|
||||
@ -624,7 +623,9 @@ static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size
|
||||
mi_assert_internal(slice->slice_count == slice_count); // no overflow?
|
||||
slice->slice_offset = 0;
|
||||
if (slice_count > 1) {
|
||||
mi_slice_t* last = &segment->slices[slice_index + slice_count - 1];
|
||||
mi_slice_t* last = slice + slice_count - 1;
|
||||
mi_slice_t* end = (mi_slice_t*)mi_segment_slices_end(segment);
|
||||
if (last > end) { last = end; }
|
||||
last->slice_count = 0;
|
||||
last->slice_offset = (uint32_t)(sizeof(mi_page_t)*(slice_count - 1));
|
||||
last->block_size = 0;
|
||||
@ -808,7 +809,7 @@ static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_aren
|
||||
----------------------------------------------------------- */
|
||||
|
||||
static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id,
|
||||
size_t* psegment_slices, size_t* ppre_size, size_t* pinfo_slices,
|
||||
size_t* psegment_slices, size_t* pinfo_slices,
|
||||
bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
|
||||
|
||||
{
|
||||
@ -825,7 +826,7 @@ static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment
|
||||
align_offset = _mi_align_up( info_size, MI_SEGMENT_ALIGN );
|
||||
const size_t extra = align_offset - info_size;
|
||||
// recalculate due to potential guard pages
|
||||
*psegment_slices = mi_segment_calculate_slices(required + extra, ppre_size, pinfo_slices);
|
||||
*psegment_slices = mi_segment_calculate_slices(required + extra, pinfo_slices);
|
||||
mi_assert_internal(*psegment_slices > 0 && *psegment_slices <= UINT32_MAX);
|
||||
}
|
||||
|
||||
@ -874,8 +875,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
|
||||
|
||||
// calculate needed sizes first
|
||||
size_t info_slices;
|
||||
size_t pre_size;
|
||||
size_t segment_slices = mi_segment_calculate_slices(required, &pre_size, &info_slices);
|
||||
size_t segment_slices = mi_segment_calculate_slices(required, &info_slices);
|
||||
mi_assert_internal(segment_slices > 0 && segment_slices <= UINT32_MAX);
|
||||
|
||||
// Commit eagerly only if not the first N lazy segments (to reduce impact of many threads that allocate just a little)
|
||||
@ -887,7 +887,7 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
|
||||
|
||||
// Allocate the segment from the OS
|
||||
mi_segment_t* segment = mi_segment_os_alloc(required, page_alignment, eager_delay, req_arena_id,
|
||||
&segment_slices, &pre_size, &info_slices, commit, tld, os_tld);
|
||||
&segment_slices, &info_slices, commit, tld, os_tld);
|
||||
if (segment == NULL) return NULL;
|
||||
|
||||
// zero the segment info? -- not always needed as it may be zero initialized from the OS
|
||||
@ -916,7 +916,6 @@ static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi
|
||||
// in secure mode, we set up a protected page in between the segment info
|
||||
// and the page data, and at the end of the segment.
|
||||
size_t os_pagesize = _mi_os_page_size();
|
||||
mi_assert_internal(mi_segment_info_size(segment) - os_pagesize >= pre_size);
|
||||
_mi_os_protect((uint8_t*)segment + mi_segment_info_size(segment) - os_pagesize, os_pagesize);
|
||||
uint8_t* end = (uint8_t*)segment + mi_segment_size(segment) - os_pagesize;
|
||||
mi_segment_ensure_committed(segment, end, os_pagesize, tld->stats);
|
||||
@ -1007,11 +1006,13 @@ static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld
|
||||
_mi_os_reset(start, psize, tld->stats);
|
||||
}
|
||||
|
||||
// zero the page data, but not the segment fields
|
||||
// zero the page data, but not the segment fields and heap tag
|
||||
page->is_zero_init = false;
|
||||
uint8_t heap_tag = page->heap_tag;
|
||||
ptrdiff_t ofs = offsetof(mi_page_t, capacity);
|
||||
_mi_memzero((uint8_t*)page + ofs, sizeof(*page) - ofs);
|
||||
page->block_size = 1;
|
||||
page->heap_tag = heap_tag;
|
||||
|
||||
// and free it
|
||||
mi_slice_t* slice = mi_segment_span_free_coalesce(mi_page_to_slice(page), tld);
|
||||
@ -1212,8 +1213,13 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap,
|
||||
mi_assert_internal(page->next == NULL && page->prev==NULL);
|
||||
_mi_stat_decrease(&tld->stats->pages_abandoned, 1);
|
||||
segment->abandoned--;
|
||||
// set the heap again and allow delayed free again
|
||||
mi_page_set_heap(page, heap);
|
||||
// set the heap again and allow heap thread delayed free again.
|
||||
mi_heap_t* target_heap = _mi_heap_by_tag(heap, page->heap_tag); // allow custom heaps to separate objects
|
||||
if (target_heap == NULL) {
|
||||
target_heap = heap;
|
||||
_mi_error_message(EINVAL, "page with tag %u cannot be reclaimed by a heap with the same tag (using %u instead)\n", page->heap_tag, heap->tag );
|
||||
}
|
||||
mi_page_set_heap(page, target_heap);
|
||||
_mi_page_use_delayed_free(page, MI_USE_DELAYED_FREE, true); // override never (after heap is set)
|
||||
_mi_page_free_collect(page, false); // ensure used count is up to date
|
||||
if (mi_page_all_free(page)) {
|
||||
@ -1222,8 +1228,8 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap,
|
||||
}
|
||||
else {
|
||||
// otherwise reclaim it into the heap
|
||||
_mi_page_reclaim(heap, page);
|
||||
if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page)) {
|
||||
_mi_page_reclaim(target_heap, page);
|
||||
if (requested_block_size == mi_page_block_size(page) && mi_page_has_any_available(page) && heap == target_heap) {
|
||||
if (right_page_reclaimed != NULL) { *right_page_reclaimed = true; }
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user