/* ---------------------------------------------------------------------------- Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #pragma once #ifndef MIMALLOC_INTERNAL_H #define MIMALLOC_INTERNAL_H // -------------------------------------------------------------------------- // This file contains the internal API's of mimalloc and various utility // functions and macros. // -------------------------------------------------------------------------- #include "types.h" #include "track.h" #if (MI_DEBUG>0) #define mi_trace_message … #else #define mi_trace_message(...) … #endif #if defined(__EMSCRIPTEN__) && !defined(__wasi__) #define __wasi__ #endif #if defined(__cplusplus) #define mi_decl_externc … #else #define mi_decl_externc #endif // pthreads #if !defined(_WIN32) && !defined(__wasi__) #define MI_USE_PTHREADS #include <pthread.h> #endif // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); void _mi_warning_message(const char* fmt, ...); void _mi_verbose_message(const char* fmt, ...); void _mi_trace_message(const char* fmt, ...); void _mi_options_init(void); void _mi_error_message(int err, const char* fmt, ...); // random.c void _mi_random_init(mi_random_ctx_t* ctx); void _mi_random_init_weak(mi_random_ctx_t* ctx); void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); uintptr_t _mi_random_next(mi_random_ctx_t* ctx); uintptr_t _mi_heap_random_next(mi_heap_t* heap); uintptr_t _mi_os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); // init.c extern mi_decl_cache_align mi_stats_t _mi_stats_main; extern mi_decl_cache_align const mi_page_t _mi_page_empty; bool _mi_is_main_thread(void); size_t _mi_current_thread_count(void); bool _mi_preloading(void); // true while the C runtime is not initialized yet mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap void _mi_thread_done(mi_heap_t* heap); void _mi_thread_data_collect(void); void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); // os.c void _mi_os_init(void); // called from process init void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* stats); void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* stats); void _mi_os_free_ex(void* p, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* stats); size_t _mi_os_page_size(void); size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); bool _mi_os_has_virtual_reserve(void); bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); bool _mi_os_reset(void* addr, size_t size, mi_stats_t* tld_stats); bool _mi_os_commit(void* p, size_t size, bool* is_zero, mi_stats_t* stats); bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_purge(void* p, size_t size, mi_stats_t* stats); bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats); void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* stats); void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats); void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size); bool _mi_os_use_large_page(size_t size, size_t alignment); size_t _mi_os_large_page_size(void); void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_secs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid); // arena.c mi_arena_id_t _mi_arena_id_none(void); void _mi_arena_free(void* p, size_t size, size_t still_committed_size, mi_memid_t memid, mi_stats_t* stats); void* _mi_arena_alloc(size_t size, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); void* _mi_arena_alloc_aligned(size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_large, mi_arena_id_t req_arena_id, mi_memid_t* memid, mi_os_tld_t* tld); bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_id_t request_arena_id); bool _mi_arena_contains(const void* p); void _mi_arena_collect(bool force_purge, mi_stats_t* stats); void _mi_arena_unsafe_destroy_all(mi_stats_t* stats); // "segment-map.c" void _mi_segment_map_allocated_at(const mi_segment_t* segment); void _mi_segment_map_freed_at(const mi_segment_t* segment); // "segment.c" extern mi_abandoned_pool_t _mi_abandoned_default; // global abandoned pool mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); void _mi_segment_thread_collect(mi_segments_tld_t* tld); bool _mi_abandoned_pool_visit_blocks(mi_abandoned_pool_t* pool, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); #if MI_HUGE_PAGE_ABANDON void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); #else void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block); #endif uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size); // page start for any page void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); void _mi_abandoned_await_readers(mi_abandoned_pool_t *pool); void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld); // "page.c" void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... void _mi_heap_delayed_free_all(mi_heap_t* heap); bool _mi_heap_delayed_free_partial(mi_heap_t* heap); void _mi_heap_collect_retired(mi_heap_t* heap, bool force); void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); bool _mi_page_try_use_delayed_free(mi_page_t* page, mi_delayed_t delay, bool override_never); size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); void _mi_deferred_free(mi_heap_t* heap, bool force); void _mi_page_free_collect(mi_page_t* page,bool force); void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id, bool no_reclaim, uint8_t tag); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); bool _mi_heap_memid_is_suitable(mi_heap_t* heap, mi_memid_t memid); void _mi_heap_unsafe_destroy_all(void); void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); bool _mi_heap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t *page, mi_block_visit_fun* visitor, void* arg); // "stats.c" void _mi_stats_done(mi_stats_t* stats); mi_msecs_t _mi_clock_now(void); mi_msecs_t _mi_clock_end(mi_msecs_t start); mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_malloc_generic` void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) mi_attr_noexcept; void* _mi_heap_malloc_zero_ex(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept; // called from `_mi_heap_malloc_aligned` void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); bool _mi_free_delayed_block(mi_block_t* block); void _mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool is_local, void* p) mi_attr_noexcept; // for runtime integration void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); // option.c, c primitives char _mi_toupper(char c); int _mi_strnicmp(const char* s, const char* t, size_t n); void _mi_strlcpy(char* dest, const char* src, size_t dest_size); void _mi_strlcat(char* dest, const char* src, size_t dest_size); size_t _mi_strlen(const char* s); size_t _mi_strnlen(const char* s, size_t max_len); #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); #endif // ------------------------------------------------------ // Branches // ------------------------------------------------------ #if defined(__GNUC__) || defined(__clang__) #define mi_unlikely(x) … #define mi_likely(x) … #elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L) #define mi_unlikely … #define mi_likely … #else #define mi_unlikely … #define mi_likely … #endif #ifndef __has_builtin #define __has_builtin … #endif /* ----------------------------------------------------------- Error codes passed to `_mi_fatal_error` All are recoverable but EFAULT is a serious error and aborts by default in secure mode. For portability define undefined error codes using common Unix codes: <https://www-numi.fnal.gov/offline_software/srt_public_context/WebDocs/Errors/unix_system_errors.html> ----------------------------------------------------------- */ #include <errno.h> #ifndef EAGAIN // double free #define EAGAIN … #endif #ifndef ENOMEM // out of memory #define ENOMEM … #endif #ifndef EFAULT // corrupted free-list or meta-data #define EFAULT … #endif #ifndef EINVAL // trying to free an invalid pointer #define EINVAL … #endif #ifndef EOVERFLOW // count*size overflow #define EOVERFLOW … #endif /* ----------------------------------------------------------- Inlined definitions ----------------------------------------------------------- */ #define MI_UNUSED(x) … #if (MI_DEBUG>0) #define MI_UNUSED_RELEASE … #else #define MI_UNUSED_RELEASE(x) … #endif #define MI_INIT4(x) … #define MI_INIT8(x) … #define MI_INIT16(x) … #define MI_INIT32(x) … #define MI_INIT64(x) … #define MI_INIT128(x) … #define MI_INIT256(x) … #include <string.h> // initialize a local variable to zero; use memset as compilers optimize constant sized memset's #define _mi_memzero_var(x) … // Is `x` a power of two? (0 is considered a power of two) static inline bool _mi_is_power_of_two(uintptr_t x) { … } // Is a pointer aligned? static inline bool _mi_is_aligned(void* p, size_t alignment) { … } // Align upwards static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { … } // Align downwards static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { … } // Divide upwards: `s <= _mi_divide_up(s,d)*d < s+d`. static inline uintptr_t _mi_divide_up(uintptr_t size, size_t divider) { … } // Is memory zero initialized? static inline bool mi_mem_is_zero(const void* p, size_t size) { … } // Align a byte size to a size in _machine words_, // i.e. byte size == `wsize*sizeof(void*)`. static inline size_t _mi_wsize_from_size(size_t size) { … } // Overflow detecting multiply #if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include <limits.h> // UINT_MAX, ULONG_MAX #if defined(_CLOCK_T) // for Illumos #undef _CLOCK_T #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { … } #else /* __builtin_umul_overflow is unavailable */ static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { #define MI_MUL_NO_OVERFLOW … *total = count * size; // note: gcc/clang optimize this to directly check the overflow flag return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) && size > 0 && (SIZE_MAX / size) < count); } #endif // Safe multiply `count*size` into `total`; return `true` on overflow. static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* total) { … } /*---------------------------------------------------------------------------------------- Heap functions ------------------------------------------------------------------------------------------- */ extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap static inline bool mi_heap_is_backing(const mi_heap_t* heap) { … } static inline bool mi_heap_is_initialized(mi_heap_t* heap) { … } static inline uintptr_t _mi_ptr_cookie(const void* p) { … } /* ----------------------------------------------------------- Pages ----------------------------------------------------------- */ static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { … } // Segment that contains the pointer // Large aligned blocks may be aligned at N*MI_SEGMENT_SIZE (inside a huge segment > MI_SEGMENT_SIZE), // and we need align "down" to the segment info which is `MI_SEGMENT_SIZE` bytes before it; // therefore we align one byte before `p`. static inline mi_segment_t* _mi_ptr_segment(const void* p) { … } static inline mi_page_t* mi_slice_to_page(mi_slice_t* s) { … } static inline mi_slice_t* mi_page_to_slice(mi_page_t* p) { … } // Segment belonging to a page static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { … } static inline mi_slice_t* mi_slice_first(const mi_slice_t* slice) { … } // Get the page containing the pointer (performance critical as it is called in mi_free) static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { … } // Quick page start for initialized pages static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { … } // Get the page containing the pointer static inline mi_page_t* _mi_ptr_page(void* p) { … } // Get the block size of a page (special case for huge objects) static inline size_t mi_page_block_size(const mi_page_t* page) { … } static inline bool mi_page_is_huge(const mi_page_t* page) { … } // Get the usable block size of a page without fixed padding. // This may still include internal padding due to alignment and rounding up size classes. static inline size_t mi_page_usable_block_size(const mi_page_t* page) { … } // size of a segment static inline size_t mi_segment_size(mi_segment_t* segment) { … } static inline uint8_t* mi_segment_end(mi_segment_t* segment) { … } // Thread free access static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { … } static inline mi_delayed_t mi_page_thread_free_flag(const mi_page_t* page) { … } // Heap access static inline mi_heap_t* mi_page_heap(const mi_page_t* page) { … } static inline void mi_page_set_heap(mi_page_t* page, mi_heap_t* heap) { … } // Thread free flag helpers static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { … } static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) { … } static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) { … } static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) { … } static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) { … } // are all blocks in a page freed? // note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. static inline bool mi_page_all_free(const mi_page_t* page) { … } // are there any available blocks? static inline bool mi_page_has_any_available(const mi_page_t* page) { … } // are there immediately available blocks, i.e. blocks available on the free list. static inline bool mi_page_immediate_available(const mi_page_t* page) { … } // is more than 7/8th of a page in use? static inline bool mi_page_mostly_used(const mi_page_t* page) { … } static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { … } //----------------------------------------------------------- // Page flags //----------------------------------------------------------- static inline bool mi_page_is_in_full(const mi_page_t* page) { … } static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { … } static inline bool mi_page_has_aligned(const mi_page_t* page) { … } static inline void mi_page_set_has_aligned(mi_page_t* page, bool has_aligned) { … } /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers This is to protect against buffer overflow exploits where the free list is mutated. Many hardened allocators xor the next pointer `p` with a secret key `k1`, as `p^k1`. This prevents overwriting with known values but might be still too weak: if the attacker can guess the pointer `p` this can reveal `k1` (since `p^k1^p == k1`). Moreover, if multiple blocks can be read as well, the attacker can xor both as `(p1^k1) ^ (p2^k1) == p1^p2` which may reveal a lot about the pointers (and subsequently `k1`). Instead mimalloc uses an extra key `k2` and encodes as `((p^k2)<<<k1)+k1`. Since these operations are not associative, the above approaches do not work so well any more even if the `p` can be guesstimated. For example, for the read case we can subtract two entries to discard the `+k1` term, but that leads to `((p1^k2)<<<k1) - ((p2^k2)<<<k1)` at best. We include the left-rotation since xor and addition are otherwise linear in the lowest bit. Finally, both keys are unique per page which reduces the re-use of keys by a large factor. We also pass a separate `null` value to be used as `NULL` or otherwise `(k2<<<k1)+k1` would appear (too) often as a sentinel value. ------------------------------------------------------------------- */ static inline bool mi_is_in_same_segment(const void* p, const void* q) { … } static inline bool mi_is_in_same_page(const void* p, const void* q) { … } static inline uintptr_t mi_rotl(uintptr_t x, uintptr_t shift) { … } static inline uintptr_t mi_rotr(uintptr_t x, uintptr_t shift) { … } static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { … } static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { … } static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { … } static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { … } static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { … } static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) { … } // ------------------------------------------------------------------- // commit mask // ------------------------------------------------------------------- static inline void mi_commit_mask_create_empty(mi_commit_mask_t* cm) { … } static inline void mi_commit_mask_create_full(mi_commit_mask_t* cm) { … } static inline bool mi_commit_mask_is_empty(const mi_commit_mask_t* cm) { … } static inline bool mi_commit_mask_is_full(const mi_commit_mask_t* cm) { … } // defined in `segment.c`: size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total); size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx); #define mi_commit_mask_foreach(cm,idx,count) … #define mi_commit_mask_foreach_end() … /* ----------------------------------------------------------- memory id's ----------------------------------------------------------- */ static inline mi_memid_t _mi_memid_create(mi_memkind_t memkind) { … } static inline mi_memid_t _mi_memid_none(void) { … } static inline mi_memid_t _mi_memid_create_os(bool committed, bool is_zero, bool is_large) { … } // ------------------------------------------------------------------- // Fast "random" shuffle // ------------------------------------------------------------------- static inline uintptr_t _mi_random_shuffle(uintptr_t x) { … } // ------------------------------------------------------------------- // Optimize numa node access for the common case (= one node) // ------------------------------------------------------------------- int _mi_os_numa_node_get(mi_os_tld_t* tld); size_t _mi_os_numa_node_count_get(void); extern _Atomic(size_t) _mi_numa_node_count; static inline int _mi_os_numa_node(mi_os_tld_t* tld) { … } static inline size_t _mi_os_numa_node_count(void) { … } // ----------------------------------------------------------------------- // Count bits: trailing or leading zeros (with MI_INTPTR_BITS on all zero) // ----------------------------------------------------------------------- #if defined(__GNUC__) #include <limits.h> // LONG_MAX #define MI_HAVE_FAST_BITSCAN static inline size_t mi_clz(uintptr_t x) { … } static inline size_t mi_ctz(uintptr_t x) { … } #elif defined(_MSC_VER) #include <limits.h> // LONG_MAX #include <intrin.h> // BitScanReverse64 #define MI_HAVE_FAST_BITSCAN static inline size_t mi_clz(uintptr_t x) { if (x==0) return MI_INTPTR_BITS; unsigned long idx; #if (INTPTR_MAX == LONG_MAX) _BitScanReverse(&idx, x); #else _BitScanReverse64(&idx, x); #endif return ((MI_INTPTR_BITS - 1) - idx); } static inline size_t mi_ctz(uintptr_t x) { if (x==0) return MI_INTPTR_BITS; unsigned long idx; #if (INTPTR_MAX == LONG_MAX) _BitScanForward(&idx, x); #else _BitScanForward64(&idx, x); #endif return idx; } #else static inline size_t mi_ctz32(uint32_t x) { // de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf> static const unsigned char debruijn[32] = { 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; if (x==0) return 32; return debruijn[((x & -(int32_t)x) * 0x077CB531UL) >> 27]; } static inline size_t mi_clz32(uint32_t x) { // de Bruijn multiplication, see <http://supertech.csail.mit.edu/papers/debruijn.pdf> static const uint8_t debruijn[32] = { 31, 22, 30, 21, 18, 10, 29, 2, 20, 17, 15, 13, 9, 6, 28, 1, 23, 19, 11, 3, 16, 14, 7, 24, 12, 4, 8, 25, 5, 26, 27, 0 }; if (x==0) return 32; x |= x >> 1; x |= x >> 2; x |= x >> 4; x |= x >> 8; x |= x >> 16; return debruijn[(uint32_t)(x * 0x07C4ACDDUL) >> 27]; } static inline size_t mi_clz(uintptr_t x) { if (x==0) return MI_INTPTR_BITS; #if (MI_INTPTR_BITS <= 32) return mi_clz32((uint32_t)x); #else size_t count = mi_clz32((uint32_t)(x >> 32)); if (count < 32) return count; return (32 + mi_clz32((uint32_t)x)); #endif } static inline size_t mi_ctz(uintptr_t x) { if (x==0) return MI_INTPTR_BITS; #if (MI_INTPTR_BITS <= 32) return mi_ctz32((uint32_t)x); #else size_t count = mi_ctz32((uint32_t)x); if (count < 32) return count; return (32 + mi_ctz32((uint32_t)(x>>32))); #endif } #endif // "bit scan reverse": Return index of the highest bit (or MI_INTPTR_BITS if `x` is zero) static inline size_t mi_bsr(uintptr_t x) { … } // --------------------------------------------------------------------------------- // Provide our own `_mi_memcpy` for potential performance optimizations. // // For now, only on Windows with msvc/clang-cl we optimize to `rep movsb` if // we happen to run on x86/x64 cpu's that have "fast short rep movsb" (FSRM) support // (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. // --------------------------------------------------------------------------------- #if !MI_TRACK_ENABLED && defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) #include <intrin.h> extern bool _mi_cpu_has_fsrm; static inline void _mi_memcpy(void* dst, const void* src, size_t n) { if (_mi_cpu_has_fsrm) { __movsb((unsigned char*)dst, (const unsigned char*)src, n); } else { memcpy(dst, src, n); } } static inline void _mi_memzero(void* dst, size_t n) { if (_mi_cpu_has_fsrm) { __stosb((unsigned char*)dst, 0, n); } else { memset(dst, 0, n); } } #else static inline void _mi_memcpy(void* dst, const void* src, size_t n) { … } static inline void _mi_memzero(void* dst, size_t n) { … } #endif // ------------------------------------------------------------------------------- // The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned // This is used for example in `mi_realloc`. // ------------------------------------------------------------------------------- #if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // On GCC/CLang we provide a hint that the pointers are word aligned. static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { … } static inline void _mi_memzero_aligned(void* dst, size_t n) { … } #else // Default fallback on `_mi_memcpy` static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); _mi_memcpy(dst, src, n); } static inline void _mi_memzero_aligned(void* dst, size_t n) { mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); _mi_memzero(dst, n); } #endif #endif
Codebase Browser
cpython