diff --git a/include/mimalloc.h b/include/mimalloc.h index 800cfd7e..fbc9887d 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -345,7 +345,8 @@ typedef enum mi_option_e { mi_option_allow_decommit, mi_option_segment_decommit_delay, mi_option_decommit_extend_delay, - mi_option_destroy_on_exit, + mi_option_destroy_on_exit, + mi_option_eager_reserve, _mi_option_last } mi_option_t; @@ -356,8 +357,9 @@ mi_decl_export void mi_option_disable(mi_option_t option); mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); -mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); -mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); +mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); diff --git a/src/arena.c b/src/arena.c index 43defe00..8971a2a9 100644 --- a/src/arena.c +++ b/src/arena.c @@ -181,6 +181,28 @@ static mi_decl_noinline void* mi_arena_alloc_from(mi_arena_t* arena, size_t aren return p; } +// allocate in a speficic arena +static void* mi_arena_alloc_in(mi_arena_id_t arena_id, int numa_node, size_t size, size_t alignment, + bool* commit, bool* large, bool* is_pinned, bool* is_zero, + mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld ) +{ + MI_UNUSED_RELEASE(alignment); + mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); + const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); + const size_t bcount = mi_block_count_of_size(size); + const size_t arena_index = mi_arena_id_index(arena_id); + mi_assert_internal(arena_index < max_arena); + mi_assert_internal(size <= bcount * MI_ARENA_BLOCK_SIZE); + if (arena_index >= max_arena) return NULL; + + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[arena_index]); + if (arena == NULL) return NULL; + if (arena->numa_node >= 0 && arena->numa_node != numa_node) return NULL; + if (!(*large) && arena->is_large) return NULL; + return mi_arena_alloc_from(arena, arena_index, bcount, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); +} + + // allocate from an arena with fallback to the OS static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, @@ -253,6 +275,20 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) { void* p = mi_arena_allocate(numa_node, size, alignment, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); if (p != NULL) return p; + + // otherwise, try to first eagerly reserve a new arena + size_t eager_reserve = mi_option_get_size(mi_option_eager_reserve); + eager_reserve = _mi_align_up(eager_reserve, MI_ARENA_BLOCK_SIZE); + if (eager_reserve > 0 && eager_reserve >= size && // eager reserve enabled and large enough? + req_arena_id == _mi_arena_id_none() && // not exclusive? + mi_atomic_load_relaxed(&mi_arena_count) < 3*(MI_MAX_ARENAS/4) ) // not too many arenas already? + { + mi_arena_id_t arena_id = 0; + if (mi_reserve_os_memory_ex(eager_reserve, false /* commit */, *large /* allow large*/, false /* exclusive */, &arena_id) == 0) { + p = mi_arena_alloc_in(arena_id, numa_node, size, alignment, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); + if (p != NULL) return p; + } + } } // finally, fall back to the OS diff --git a/src/options.c b/src/options.c index c39e2052..8f907851 100644 --- a/src/options.c +++ b/src/options.c @@ -87,7 +87,12 @@ static mi_option_desc_t options[_mi_option_last] = { 1, UNINIT, MI_OPTION(allow_decommit) }, // decommit slices when no longer used (after decommit_delay milli-seconds) { 500, UNINIT, MI_OPTION(segment_decommit_delay) }, // decommit delay in milli-seconds for freed segments { 1, UNINIT, MI_OPTION(decommit_extend_delay) }, - { 0, UNINIT, MI_OPTION(destroy_on_exit)} // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + #if (MI_INTPTR_SIZE>4) + { 1024L*1024L, UNINIT, MI_OPTION(eager_reserve) } // reserve memory N KiB at a time + #else + { 128L*1024L, UNINIT, MI_OPTION(eager_reserve) } + #endif }; static void mi_option_init(mi_option_desc_t* desc); @@ -125,6 +130,12 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma return (x < min ? min : (x > max ? max : x)); } +mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { + mi_assert_internal(option == mi_option_reserve_os_memory || option == mi_option_eager_reserve); + long x = mi_option_get(option); + return (x < 0 ? 0 : (size_t)x * MI_KiB); +} + void mi_option_set(mi_option_t option, long value) { mi_assert(option >= 0 && option < _mi_option_last); if (option < 0 || option >= _mi_option_last) return; @@ -527,7 +538,7 @@ static void mi_option_init(mi_option_desc_t* desc) { else { char* end = buf; long value = strtol(buf, &end, 10); - if (desc->option == mi_option_reserve_os_memory) { + if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_eager_reserve) { // this option is interpreted in KiB to prevent overflow of `long` if (*end == 'K') { end++; } else if (*end == 'M') { value *= MI_KiB; end++; }