folly/folly/memory/JemallocHugePageAllocator.cpp

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/memory/JemallocHugePageAllocator.h>

#include <sstream>

#include <folly/CPortability.h>
#include <folly/memory/Malloc.h>
#include <folly/portability/Malloc.h>
#include <folly/portability/String.h>
#include <folly/portability/SysMman.h>
#include <folly/portability/SysTypes.h>

#include <glog/logging.h>

#if (defined(MADV_HUGEPAGE) || defined(MAP_ALIGNED_SUPER)) && \
    defined(FOLLY_USE_JEMALLOC) && !FOLLY_SANITIZE

#if defined(__FreeBSD__) || (JEMALLOC_VERSION_MAJOR >= 5)
#define FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED …
#else
#define FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED …
#endif // defined(__FreeBSD__) || (JEMALLOC_VERSION_MAJOR >= 5)

#else
#define FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED …
#endif // MADV_HUGEPAGE || MAP_ALIGNED_SUPER && defined(FOLLY_USE_JEMALLOC) &&
       // !FOLLY_SANITIZE

#if !FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
// Some mocks when jemalloc.h is not included or version too old
// or when the system does not support the MADV_HUGEPAGE madvise flag
#undef MALLOCX_ARENA
#undef MALLOCX_TCACHE_NONE
#undef MADV_HUGEPAGE
#define MALLOCX_ARENA(x) …
#define MALLOCX_TCACHE_NONE …
#define MADV_HUGEPAGE …

#if !defined(JEMALLOC_VERSION_MAJOR) || (JEMALLOC_VERSION_MAJOR < 5)
extent_hooks_t;
extent_alloc_t;
struct extent_hooks_s { … };
#endif // JEMALLOC_VERSION_MAJOR

#endif // FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED

namespace folly {
namespace {

void print_error(int err, const char* msg) { … }

class HugePageArena { … };

constexpr size_t kHugePageSize = …;

// Singleton arena instance
HugePageArena arena;

template <typename T, typename U>
static inline T align_up(T val, U alignment) { … }

// mmap enough memory to hold the aligned huge pages, then use madvise
// to get huge pages. This can be checked in /proc/<pid>/smaps.
// If successful, sets the arena member pointers to reflect the mapped memory.
// Otherwise, leaves them unchanged (zeroed).
void HugePageArena::map_pages(size_t initial_nr_pages, size_t max_nr_pages) { … }

void HugePageArena::init_more(int nr_pages) { … }

// Warning: This can be called inside malloc(). Check the comments in
// HugePageArena::allocHook to understand the restrictions that imposes before
// making any change to this function.
// Requirement: upto > freePtr_ && upto > protEnd_.
// Returns whether the setup succeeded.
bool HugePageArena::setup_next_pages(uintptr_t upto) { … }

// WARNING WARNING WARNING
// This function is the hook invoked on malloc path for the hugepage allocator.
// This means it should not, itself, call malloc. If any of the following
// happens within this function, it *WILL* cause a DEADLOCK (from the circular
// dependency):
// - any dynamic memory allocation (i.e. calls to malloc)
// - any operations that may lead to dynamic operations, such as logging (e.g.
//   LOG, VLOG, LOG_IF) and DCHECK.
// WARNING WARNING WARNING
void* HugePageArena::allocHook(
    extent_hooks_t* extent,
    void* new_addr,
    size_t size,
    size_t alignment,
    bool* zero,
    bool* commit,
    unsigned arena_ind) { … }

int HugePageArena::init(int initial_nr_pages, int max_nr_pages) { … }

// Warning: Check the comments in HugePageArena::allocHook before making any
// change to this function.
void* HugePageArena::reserve(size_t size, size_t alignment) { … }

} // namespace

int JemallocHugePageAllocator::flags_{ … };

bool JemallocHugePageAllocator::default_init() { … }

bool JemallocHugePageAllocator::init(int initial_nr_pages, int max_nr_pages) { … }

void* JemallocHugePageAllocator::allocate(size_t size) { … }

void* JemallocHugePageAllocator::reallocate(void* p, size_t size) { … }

void JemallocHugePageAllocator::deallocate(void* p, size_t) { … }

bool JemallocHugePageAllocator::initialized() { … }

size_t JemallocHugePageAllocator::freeSpace() { … }

bool JemallocHugePageAllocator::addressInArena(void* address) { … }

bool JemallocHugePageAllocator::hugePagesAllocSupported() { … }

unsigned arenaIndex() { … }

} // namespace folly