From 6eee9e4be20acb23930bac7e29c5d589aaf164a7 Mon Sep 17 00:00:00 2001 From: Daan Leijen Date: Thu, 30 Mar 2023 16:20:11 -0700 Subject: [PATCH] add option eager_reserve --- include/mimalloc.h | 6 ++++-- src/arena.c | 36 ++++++++++++++++++++++++++++++++++++ src/options.c | 11 +++++++++-- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 564be236..c7b6d61e 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -339,6 +339,7 @@ typedef enum mi_option_e { mi_option_max_warnings, mi_option_max_segment_reclaim, mi_option_destroy_on_exit, + mi_option_eager_reserve, _mi_option_last } mi_option_t; @@ -349,8 +350,9 @@ mi_decl_export void mi_option_disable(mi_option_t option); mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); -mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); -mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); +mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); +mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); mi_decl_export void mi_option_set(mi_option_t option, long value); mi_decl_export void mi_option_set_default(mi_option_t option, long value); diff --git a/src/arena.c b/src/arena.c index 152f7bea..f92cf683 100644 --- a/src/arena.c +++ b/src/arena.c @@ -191,6 +191,28 @@ static void* mi_arena_alloc_from(mi_arena_t* arena, size_t arena_index, size_t n return p; } +// allocate in a speficic arena +static void* mi_arena_alloc_in(mi_arena_id_t arena_id, int numa_node, size_t size, size_t alignment, + bool* commit, bool* large, bool* is_pinned, bool* is_zero, + mi_arena_id_t req_arena_id, size_t* memid, mi_os_tld_t* tld ) +{ + MI_UNUSED_RELEASE(alignment); + mi_assert_internal(alignment <= MI_SEGMENT_ALIGN); + const size_t max_arena = mi_atomic_load_relaxed(&mi_arena_count); + const size_t bcount = mi_block_count_of_size(size); + const size_t arena_index = mi_arena_id_index(arena_id); + mi_assert_internal(arena_index < max_arena); + mi_assert_internal(size <= bcount * MI_ARENA_BLOCK_SIZE); + if (arena_index >= max_arena) return NULL; + + mi_arena_t* arena = mi_atomic_load_ptr_relaxed(mi_arena_t, &mi_arenas[arena_index]); + if (arena == NULL) return NULL; + if (arena->numa_node >= 0 && arena->numa_node != numa_node) return NULL; + if (!(*large) && arena->is_large) return NULL; + return mi_arena_alloc_from(arena, arena_index, bcount, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); +} + + // allocate from an arena with fallback to the OS static mi_decl_noinline void* mi_arena_allocate(int numa_node, size_t size, size_t alignment, bool* commit, bool* large, bool* is_pinned, bool* is_zero, @@ -263,6 +285,20 @@ void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset if (size >= MI_ARENA_MIN_OBJ_SIZE && alignment <= MI_SEGMENT_ALIGN && align_offset == 0) { void* p = mi_arena_allocate(numa_node, size, alignment, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); if (p != NULL) return p; + + // otherwise, try to first eagerly reserve a new arena + size_t eager_reserve = mi_option_get_size(mi_option_eager_reserve); + eager_reserve = _mi_align_up(eager_reserve, MI_ARENA_BLOCK_SIZE); + if (eager_reserve > 0 && eager_reserve >= size && // eager reserve enabled and large enough? + req_arena_id == _mi_arena_id_none() && // not exclusive? + mi_atomic_load_relaxed(&mi_arena_count) < 3*(MI_MAX_ARENAS/4) ) // not too many arenas already? + { + mi_arena_id_t arena_id = 0; + if (mi_reserve_os_memory_ex(eager_reserve, false /* commit */, *large /* allow large*/, false /* exclusive */, &arena_id) == 0) { + p = mi_arena_alloc_in(arena_id, numa_node, size, alignment, commit, large, is_pinned, is_zero, req_arena_id, memid, tld); + if (p != NULL) return p; + } + } } // finally, fall back to the OS diff --git a/src/options.c b/src/options.c index 6f6655f2..79b4e310 100644 --- a/src/options.c +++ b/src/options.c @@ -86,7 +86,8 @@ static mi_option_desc_t options[_mi_option_last] = { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output { 8, UNINIT, MI_OPTION(max_segment_reclaim)},// max. number of segment reclaims from the abandoned segments per try. - { 0, UNINIT, MI_OPTION(destroy_on_exit)} // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! + { 0, UNINIT, MI_OPTION(eager_reserve) } // reserve memory N KiB at a time (slower in v1.x due to regions) }; static void mi_option_init(mi_option_desc_t* desc); @@ -124,6 +125,12 @@ mi_decl_nodiscard long mi_option_get_clamp(mi_option_t option, long min, long ma return (x < min ? min : (x > max ? max : x)); } +mi_decl_nodiscard size_t mi_option_get_size(mi_option_t option) { + mi_assert_internal(option == mi_option_reserve_os_memory || option == mi_option_eager_reserve); + long x = mi_option_get(option); + return (x < 0 ? 0 : (size_t)x * MI_KiB); +} + void mi_option_set(mi_option_t option, long value) { mi_assert(option >= 0 && option < _mi_option_last); if (option < 0 || option >= _mi_option_last) return; @@ -517,7 +524,7 @@ static void mi_option_init(mi_option_desc_t* desc) { else { char* end = buf; long value = strtol(buf, &end, 10); - if (desc->option == mi_option_reserve_os_memory) { + if (desc->option == mi_option_reserve_os_memory || desc->option == mi_option_eager_reserve) { // this option is interpreted in KiB to prevent overflow of `long` if (*end == 'K') { end++; } else if (*end == 'M') { value *= MI_KiB; end++; }