#include "mimalloc.h"
#include "mimalloc/internal.h"
#include "mimalloc/atomic.h"
#include <string.h>
#include <stdio.h>
#define MI_PAGE_HUGE_ALIGN …
static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats);
static bool mi_commit_mask_all_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { … }
static bool mi_commit_mask_any_set(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm) { … }
static void mi_commit_mask_create_intersect(const mi_commit_mask_t* commit, const mi_commit_mask_t* cm, mi_commit_mask_t* res) { … }
static void mi_commit_mask_clear(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { … }
static void mi_commit_mask_set(mi_commit_mask_t* res, const mi_commit_mask_t* cm) { … }
static void mi_commit_mask_create(size_t bitidx, size_t bitcount, mi_commit_mask_t* cm) { … }
size_t _mi_commit_mask_committed_size(const mi_commit_mask_t* cm, size_t total) { … }
size_t _mi_commit_mask_next_run(const mi_commit_mask_t* cm, size_t* idx) { … }
static const mi_slice_t* mi_segment_slices_end(const mi_segment_t* segment) { … }
static uint8_t* mi_slice_start(const mi_slice_t* slice) { … }
static inline size_t mi_slice_bin8(size_t slice_count) { … }
static inline size_t mi_slice_bin(size_t slice_count) { … }
static inline size_t mi_slice_index(const mi_slice_t* slice) { … }
static void mi_span_queue_push(mi_span_queue_t* sq, mi_slice_t* slice) { … }
static mi_span_queue_t* mi_span_queue_for(size_t slice_count, mi_segments_tld_t* tld) { … }
static void mi_span_queue_delete(mi_span_queue_t* sq, mi_slice_t* slice) { … }
static bool mi_slice_is_used(const mi_slice_t* slice) { … }
#if (MI_DEBUG>=3)
static bool mi_span_queue_contains(mi_span_queue_t* sq, mi_slice_t* slice) {
for (mi_slice_t* s = sq->first; s != NULL; s = s->next) {
if (s==slice) return true;
}
return false;
}
static bool mi_segment_is_valid(mi_segment_t* segment, mi_segments_tld_t* tld) {
mi_assert_internal(segment != NULL);
mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie);
mi_assert_internal(segment->abandoned <= segment->used);
mi_assert_internal(segment->thread_id == 0 || segment->thread_id == _mi_thread_id());
mi_assert_internal(mi_commit_mask_all_set(&segment->commit_mask, &segment->purge_mask));
mi_slice_t* slice = &segment->slices[0];
const mi_slice_t* end = mi_segment_slices_end(segment);
size_t used_count = 0;
mi_span_queue_t* sq;
while(slice < end) {
mi_assert_internal(slice->slice_count > 0);
mi_assert_internal(slice->slice_offset == 0);
size_t index = mi_slice_index(slice);
size_t maxindex = (index + slice->slice_count >= segment->slice_entries ? segment->slice_entries : index + slice->slice_count) - 1;
if (mi_slice_is_used(slice)) {
used_count++;
for (size_t i = 0; i <= MI_MAX_SLICE_OFFSET && index + i <= maxindex; i++) {
mi_assert_internal(segment->slices[index + i].slice_offset == i*sizeof(mi_slice_t));
mi_assert_internal(i==0 || segment->slices[index + i].slice_count == 0);
mi_assert_internal(i==0 || segment->slices[index + i].xblock_size == 1);
}
const mi_slice_t* last = slice + slice->slice_count - 1;
if (last > slice && last < mi_segment_slices_end(segment)) {
mi_assert_internal(last->slice_offset == (slice->slice_count-1)*sizeof(mi_slice_t));
mi_assert_internal(last->slice_count == 0);
mi_assert_internal(last->xblock_size == 1);
}
}
else {
mi_slice_t* last = &segment->slices[maxindex];
if (segment->kind != MI_SEGMENT_HUGE || slice->slice_count <= (segment->slice_entries - segment->segment_info_slices)) {
mi_assert_internal((uint8_t*)slice == (uint8_t*)last - last->slice_offset);
}
mi_assert_internal(slice == last || last->slice_count == 0 );
mi_assert_internal(last->xblock_size == 0 || (segment->kind==MI_SEGMENT_HUGE && last->xblock_size==1));
if (segment->kind != MI_SEGMENT_HUGE && segment->thread_id != 0) {
sq = mi_span_queue_for(slice->slice_count,tld);
mi_assert_internal(mi_span_queue_contains(sq,slice));
}
}
slice = &segment->slices[maxindex+1];
}
mi_assert_internal(slice == end);
mi_assert_internal(used_count == segment->used + 1);
return true;
}
#endif
static size_t mi_segment_info_size(mi_segment_t* segment) { … }
static uint8_t* _mi_segment_page_start_from_slice(const mi_segment_t* segment, const mi_slice_t* slice, size_t xblock_size, size_t* page_size)
{ … }
uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size)
{ … }
static size_t mi_segment_calculate_slices(size_t required, size_t* pre_size, size_t* info_slices) { … }
static void mi_segments_track_size(long segment_size, mi_segments_tld_t* tld) { … }
static void mi_segment_os_free(mi_segment_t* segment, mi_segments_tld_t* tld) { … }
void _mi_segment_thread_collect(mi_segments_tld_t* tld) { … }
static void mi_segment_commit_mask(mi_segment_t* segment, bool conservative, uint8_t* p, size_t size, uint8_t** start_p, size_t* full_size, mi_commit_mask_t* cm) { … }
static bool mi_segment_commit(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { … }
static bool mi_segment_ensure_committed(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { … }
static bool mi_segment_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { … }
static void mi_segment_schedule_purge(mi_segment_t* segment, uint8_t* p, size_t size, mi_stats_t* stats) { … }
static void mi_segment_try_purge(mi_segment_t* segment, bool force, mi_stats_t* stats) { … }
static bool mi_segment_is_abandoned(mi_segment_t* segment) { … }
static void mi_segment_span_free(mi_segment_t* segment, size_t slice_index, size_t slice_count, bool allow_purge, mi_segments_tld_t* tld) { … }
static void mi_segment_span_remove_from_queue(mi_slice_t* slice, mi_segments_tld_t* tld) { … }
static mi_slice_t* mi_segment_span_free_coalesce(mi_slice_t* slice, mi_segments_tld_t* tld) { … }
static mi_page_t* mi_segment_span_allocate(mi_segment_t* segment, size_t slice_index, size_t slice_count, mi_segments_tld_t* tld) { … }
static void mi_segment_slice_split(mi_segment_t* segment, mi_slice_t* slice, size_t slice_count, mi_segments_tld_t* tld) { … }
static mi_page_t* mi_segments_page_find_and_allocate(size_t slice_count, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld) { … }
static mi_segment_t* mi_segment_os_alloc( size_t required, size_t page_alignment, bool eager_delayed, mi_arena_id_t req_arena_id,
size_t* psegment_slices, size_t* ppre_size, size_t* pinfo_slices,
bool commit, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{ … }
static mi_segment_t* mi_segment_alloc(size_t required, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld, mi_page_t** huge_page)
{ … }
static void mi_segment_free(mi_segment_t* segment, bool force, mi_segments_tld_t* tld) { … }
static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld);
static mi_slice_t* mi_segment_page_clear(mi_page_t* page, mi_segments_tld_t* tld) { … }
void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld)
{ … }
#define MI_TAGGED_MASK …
static mi_segment_t* mi_tagged_segment_ptr(mi_tagged_segment_t ts) { … }
static mi_tagged_segment_t mi_tagged_segment(mi_segment_t* segment, mi_tagged_segment_t ts) { … }
mi_abandoned_pool_t _mi_abandoned_default;
static void mi_abandoned_visited_push(mi_abandoned_pool_t *pool, mi_segment_t* segment) { … }
static bool mi_abandoned_visited_revisit(mi_abandoned_pool_t *pool)
{ … }
static void mi_abandoned_push(mi_abandoned_pool_t* pool, mi_segment_t* segment) { … }
void _mi_abandoned_await_readers(mi_abandoned_pool_t* pool) { … }
static mi_segment_t* mi_abandoned_pop(mi_abandoned_pool_t* pool) { … }
static void mi_segment_abandon(mi_segment_t* segment, mi_segments_tld_t* tld) { … }
void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld) { … }
static mi_slice_t* mi_slices_start_iterate(mi_segment_t* segment, const mi_slice_t** end) { … }
static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, size_t block_size, mi_segments_tld_t* tld)
{ … }
static mi_heap_t* mi_heap_by_tag(mi_heap_t* heap, uint8_t tag) { … }
static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, size_t requested_block_size, bool* right_page_reclaimed, mi_segments_tld_t* tld) { … }
void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { … }
static mi_segment_t* mi_segment_try_reclaim(mi_heap_t* heap, size_t needed_slices, size_t block_size, bool* reclaimed, mi_segments_tld_t* tld)
{ … }
void _mi_abandoned_collect(mi_heap_t* heap, bool force, mi_segments_tld_t* tld)
{ … }
static mi_segment_t* mi_segment_reclaim_or_alloc(mi_heap_t* heap, size_t needed_slices, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{ … }
static mi_page_t* mi_segments_page_alloc(mi_heap_t* heap, mi_page_kind_t page_kind, size_t required, size_t block_size, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{ … }
static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, mi_arena_id_t req_arena_id, mi_segments_tld_t* tld, mi_os_tld_t* os_tld)
{ … }
#if MI_HUGE_PAGE_ABANDON
void _mi_segment_huge_page_free(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) {
mi_assert_internal(segment->kind==MI_SEGMENT_HUGE);
mi_assert_internal(segment == _mi_page_segment(page));
mi_assert_internal(mi_atomic_load_relaxed(&segment->thread_id)==0);
mi_heap_t* heap = mi_heap_get_default();
size_t expected_tid = 0;
if (mi_atomic_cas_strong_acq_rel(&segment->thread_id, &expected_tid, heap->thread_id)) {
mi_block_set_next(page, block, page->free);
page->free = block;
page->used--;
page->is_zero = false;
mi_assert(page->used == 0);
mi_tld_t* tld = heap->tld;
_mi_segment_page_free(page, true, &tld->segments);
}
#if (MI_DEBUG!=0)
else {
mi_assert_internal(false);
}
#endif
}
#else
void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, mi_block_t* block) { … }
#endif
mi_page_t* _mi_segment_page_alloc(mi_heap_t* heap, size_t block_size, size_t page_alignment, mi_segments_tld_t* tld, mi_os_tld_t* os_tld) { … }
static bool mi_segment_visit_page(mi_segment_t* segment, mi_page_t* page, bool visit_blocks, mi_block_visit_fun* visitor, void* arg)
{ … }
static bool mi_segment_visit_pages(mi_segment_t* segment, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { … }
bool _mi_abandoned_pool_visit_blocks(mi_abandoned_pool_t* pool, uint8_t page_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { … }