/* ---------------------------------------------------------------------------- Copyright (c) 2018-2023, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ #include "mimalloc.h" #include "mimalloc/internal.h" #include "mimalloc/atomic.h" #include "mimalloc/prim.h" /* ----------------------------------------------------------- Initialization. On windows initializes support for aligned allocation and large OS pages (if MIMALLOC_LARGE_OS_PAGES is true). ----------------------------------------------------------- */ static mi_os_mem_config_t mi_os_mem_config = …; bool _mi_os_has_overcommit(void) { … } bool _mi_os_has_virtual_reserve(void) { … } // OS (small) page size size_t _mi_os_page_size(void) { … } // if large OS pages are supported (2 or 4MiB), then return the size, otherwise return the small page size (4KiB) size_t _mi_os_large_page_size(void) { … } bool _mi_os_use_large_page(size_t size, size_t alignment) { … } // round to a good OS allocation size (bounded by max 12.5% waste) size_t _mi_os_good_alloc_size(size_t size) { … } void _mi_os_init(void) { … } /* ----------------------------------------------------------- Util -------------------------------------------------------------- */ bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* stats); bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats); static void* mi_align_up_ptr(void* p, size_t alignment) { … } static void* mi_align_down_ptr(void* p, size_t alignment) { … } /* ----------------------------------------------------------- aligned hinting -------------------------------------------------------------- */ // On 64-bit systems, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate those. #if (MI_INTPTR_SIZE >= 8) static mi_decl_cache_align _Atomic(uintptr_t)aligned_base; // Return a MI_SEGMENT_SIZE aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be // properly aligned which can be more costly as it needs to be adjusted afterwards. // For a size > 1GiB this always returns NULL in order to guarantee good ASLR randomization; // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) #define MI_HINT_BASE … #define MI_HINT_AREA … #define MI_HINT_MAX … void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { … } #else void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { MI_UNUSED(try_alignment); MI_UNUSED(size); return NULL; } #endif /* ----------------------------------------------------------- Free memory -------------------------------------------------------------- */ static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats); static void mi_os_prim_free(void* addr, size_t size, bool still_committed, mi_stats_t* tld_stats) { … } void _mi_os_free_ex(void* addr, size_t size, bool still_committed, mi_memid_t memid, mi_stats_t* tld_stats) { … } void _mi_os_free(void* p, size_t size, mi_memid_t memid, mi_stats_t* tld_stats) { … } /* ----------------------------------------------------------- Primitive allocation from the OS. -------------------------------------------------------------- */ // Note: the `try_alignment` is just a hint and the returned pointer is not guaranteed to be aligned. static void* mi_os_prim_alloc(size_t size, size_t try_alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, mi_stats_t* stats) { … } // Primitive aligned allocation from the OS. // This function guarantees the allocated memory is aligned. static void* mi_os_prim_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, bool* is_large, bool* is_zero, void** base, mi_stats_t* stats) { … } /* ----------------------------------------------------------- OS API: alloc and alloc_aligned ----------------------------------------------------------- */ void* _mi_os_alloc(size_t size, mi_memid_t* memid, mi_stats_t* tld_stats) { … } void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats) { … } /* ----------------------------------------------------------- OS aligned allocation with an offset. This is used for large alignments > MI_ALIGNMENT_MAX. We use a large mimalloc page where the object can be aligned at an offset from the start of the segment. As we may need to overallocate, we need to free such pointers using `mi_free_aligned` to use the actual start of the memory region. ----------------------------------------------------------- */ void* _mi_os_alloc_aligned_at_offset(size_t size, size_t alignment, size_t offset, bool commit, bool allow_large, mi_memid_t* memid, mi_stats_t* tld_stats) { … } /* ----------------------------------------------------------- OS memory API: reset, commit, decommit, protect, unprotect. ----------------------------------------------------------- */ // OS page align within a given area, either conservative (pages inside the area only), // or not (straddling pages outside the area is possible) static void* mi_os_page_align_areax(bool conservative, void* addr, size_t size, size_t* newsize) { … } static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* newsize) { … } bool _mi_os_commit(void* addr, size_t size, bool* is_zero, mi_stats_t* tld_stats) { … } static bool mi_os_decommit_ex(void* addr, size_t size, bool* needs_recommit, mi_stats_t* tld_stats) { … } bool _mi_os_decommit(void* addr, size_t size, mi_stats_t* tld_stats) { … } // Signal to the OS that the address range is no longer in use // but may be used later again. This will release physical memory // pages and reduce swapping while keeping the memory committed. // We page align to a conservative area inside the range to reset. bool _mi_os_reset(void* addr, size_t size, mi_stats_t* stats) { … } // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, mi_stats_t* stats) { … } // either resets or decommits memory, returns true if the memory needs // to be recommitted if it is to be re-used later on. bool _mi_os_purge(void* p, size_t size, mi_stats_t * stats) { … } // Protect a region in memory to be not accessible. static bool mi_os_protectx(void* addr, size_t size, bool protect) { … } bool _mi_os_protect(void* addr, size_t size) { … } bool _mi_os_unprotect(void* addr, size_t size) { … } /* ---------------------------------------------------------------------------- Support for allocating huge OS pages (1Gib) that are reserved up-front and possibly associated with a specific NUMA node. (use `numa_node>=0`) -----------------------------------------------------------------------------*/ #define MI_HUGE_OS_PAGE_SIZE … #if (MI_INTPTR_SIZE >= 8) // To ensure proper alignment, use our own area for huge OS pages static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 // Claim an aligned address range for huge pages static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { … } #else static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { MI_UNUSED(pages); if (total_size != NULL) *total_size = 0; return NULL; } #endif // Allocate MI_SEGMENT_SIZE aligned huge pages void* _mi_os_alloc_huge_os_pages(size_t pages, int numa_node, mi_msecs_t max_msecs, size_t* pages_reserved, size_t* psize, mi_memid_t* memid) { … } // free every huge page in a range individually (as we allocated per page) // note: needed with VirtualAlloc but could potentially be done in one go on mmap'd systems. static void mi_os_free_huge_os_pages(void* p, size_t size, mi_stats_t* stats) { … } /* ---------------------------------------------------------------------------- Support NUMA aware allocation -----------------------------------------------------------------------------*/ _Atomic(size_t) _mi_numa_node_count; // = 0 // cache the node count size_t _mi_os_numa_node_count_get(void) { … } int _mi_os_numa_node_get(mi_os_tld_t* tld) { … }
Codebase Browser
cpython