From ff3bb5ce7b45272b6a97b042bbcddcd7bbc50346 Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Sun, 19 Dec 2021 14:51:15 +0100 Subject: [PATCH 1/2] Tweak documentation for mi_heap_area_t::used. For microsoft/mimalloc#503. --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 756e516c..420e0eab 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -253,7 +253,7 @@ typedef struct mi_heap_area_s { void* blocks; // start of the area containing heap blocks size_t reserved; // bytes reserved for this area (virtual) size_t committed; // current available bytes for this area - size_t used; // bytes in use by allocated blocks + size_t used; // number of allocated blocks size_t block_size; // size in bytes of each block } mi_heap_area_t; From 43e5cd2671585023d425f247b5c39cfa1207866a Mon Sep 17 00:00:00 2001 From: daan Date: Sat, 1 Jan 2022 15:54:06 -0800 Subject: [PATCH 2/2] revise assembly test for the _mi_threadid(); fix issue #495 with musl libc on arm; test specifically for bionic libc on Android --- include/mimalloc-internal.h | 122 ++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 54 deletions(-) diff --git a/include/mimalloc-internal.h b/include/mimalloc-internal.h index 06c3e95f..5f016640 100644 --- a/include/mimalloc-internal.h +++ b/include/mimalloc-internal.h @@ -698,8 +698,10 @@ static inline size_t _mi_os_numa_node_count(void) { // ------------------------------------------------------------------- // Getting the thread id should be performant as it is called in the // fast path of `_mi_free` and we specialize for various platforms. +// We only require _mi_threadid() to return a unique id for each thread. // ------------------------------------------------------------------- #if defined(_WIN32) + #define WIN32_LEAN_AND_MEAN #include static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { @@ -707,81 +709,93 @@ static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { return (uintptr_t)NtCurrentTeb(); } -#elif defined(__GNUC__) && \ - (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__)) - +// We use assembly for a fast thread id on the main platforms. The TLS layout depends on +// both the OS and libc implementation so we use specific tests for each main platform. +// If you test on another platform and it works please send a PR :-) // see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. +#elif defined(__GNUC__) && ( \ + (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__))) \ + || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__))) \ + || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ + ) + static inline void* mi_tls_slot(size_t slot) mi_attr_noexcept { void* res; const size_t ofs = (slot*sizeof(void*)); -#if defined(__i386__) - __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS -#elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS -#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI -#elif defined(__x86_64__) - __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) // arm32: defined but currently not used (see issue #495) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - res = tcb[slot]; -#elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); - tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #if defined(__i386__) + __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + res = tcb[slot]; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + res = tcb[slot]; #endif - res = tcb[slot]; -#endif return res; } -// setting a tls slot is only used on macOSX for now +// setting a tls slot is only used on macOS for now static inline void mi_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { const size_t ofs = (slot*sizeof(void*)); -#if defined(__i386__) - __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS -#elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOSX uses GS -#elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI -#elif defined(__x86_64__) - __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - tcb[slot] = value; -#elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0" : "=r" (tcb)); - tcb = (void**)((uintptr_t)tcb & ~0x07UL); // clear lower 3 bits - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #if defined(__i386__) + __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS + #elif defined(__APPLE__) && defined(__x86_64__) + __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS + #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) + __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI + #elif defined(__x86_64__) + __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS + #elif defined(__arm__) + void** tcb; MI_UNUSED(ofs); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + tcb[slot] = value; + #elif defined(__aarch64__) + void** tcb; MI_UNUSED(ofs); + #if defined(__APPLE__) // M1, issue #343 + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + tcb[slot] = value; #endif - tcb[slot] = value; -#endif } static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { -#if defined(__ANDROID__) && (defined(__arm__) || defined(__aarch64__)) - // issue #384, #495: on arm Android, slot 1 is the thread ID (pointer to pthread internal struct) - return (uintptr_t)mi_tls_slot(1); -#else - // in all our other targets, slot 0 is the pointer to the thread control block - return (uintptr_t)mi_tls_slot(0); -#endif + #if defined(__BIONIC__) + // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id + // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 + return (uintptr_t)mi_tls_slot(1); + #else + // in all our other targets, slot 0 is the thread id + // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h + // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 + return (uintptr_t)mi_tls_slot(0); + #endif } + #else -// otherwise use portable C + +// otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). static inline mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { return (uintptr_t)&_mi_heap_default; } + #endif + // ----------------------------------------------------------------------- // Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) // -----------------------------------------------------------------------