folly/folly/concurrency/CacheLocality.cpp

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <folly/concurrency/CacheLocality.h>

#ifndef _MSC_VER
#define _GNU_SOURCE …
#include <dlfcn.h>
#endif
#include <fstream>
#include <mutex>
#include <numeric>
#include <optional>

#include <fmt/core.h>
#include <glog/logging.h>
#include <folly/Indestructible.h>
#include <folly/Memory.h>
#include <folly/ScopeGuard.h>
#include <folly/detail/StaticSingletonManager.h>
#include <folly/hash/Hash.h>
#include <folly/lang/Exception.h>
#include <folly/portability/Unistd.h>
#include <folly/system/ThreadId.h>

namespace folly {

///////////// CacheLocality

/// Returns the CacheLocality information best for this machine
static CacheLocality getSystemLocalityInfo() { … }

template <>
const CacheLocality& CacheLocality::system<std::atomic>() { … }

CacheLocality::CacheLocality(std::vector<std::vector<size_t>> equivClasses) { … }

// Each level of cache has sharing sets, which are the set of cpus that share a
// common cache at that level.  These are available in a hex bitset form
// (/sys/devices/system/cpu/cpu0/cache/index0/shared_cpu_map, for example).
// They are also available in human-readable form in the shared_cpu_list file in
// the same directory.  The list is a comma-separated list of numbers and
// ranges, where the ranges are pairs of decimal numbers separated by a '-'.
//
// To sort the cpus for optimum locality we don't really need to parse the
// sharing sets, we just need a unique representative from the equivalence
// class.  The smallest value works fine, and happens to be the first decimal
// number in the file.  We load all of the equivalence class information from
// all of the cpu*/index* directories, order the cpus first by increasing
// last-level cache equivalence class, then by the smaller caches.  Finally, we
// break ties with the cpu number itself.

/// Returns the first decimal number in the line, or throws an exception if the
/// line does not start with a number terminated by ',', '-', '\n', or EOS.
static size_t parseLeadingNumber(const std::string& line) { … }

CacheLocality CacheLocality::readFromSysfsTree(
    const std::function<std::string(std::string const&)>& mapping) { … }

CacheLocality CacheLocality::readFromSysfs() { … }

namespace {

static bool procCpuinfoLineRelevant(std::string const& line) { … }

std::vector<std::tuple<size_t, size_t, size_t>> parseProcCpuinfoLines(
    std::vector<std::string> const& lines) { … }

} // namespace

CacheLocality CacheLocality::readFromProcCpuinfoLines(
    std::vector<std::string> const& lines) { … }

CacheLocality CacheLocality::readFromProcCpuinfo() { … }

CacheLocality CacheLocality::uniform(size_t numCpus) { … }

////////////// Getcpu

Getcpu::Func Getcpu::resolveVdsoFunc() { … }

/////////////// SequentialThreadId
unsigned SequentialThreadId::get() { … }

/////////////// HashingThreadId
unsigned HashingThreadId::get() { … }

namespace detail {

int AccessSpreaderBase::degenerateGetcpu(unsigned* cpu, unsigned* node, void*) { … }

struct AccessSpreaderStaticInit { … };
AccessSpreaderStaticInit AccessSpreaderStaticInit::instance;

bool AccessSpreaderBase::initialize(
    GlobalState& state,
    Getcpu::Func (&pickGetcpuFunc)(),
    const CacheLocality& (&system)()) { … }

} // namespace detail

namespace {

/**
 * A simple freelist allocator.  Allocates things of size sz, from slabs of size
 * kAllocSize.  Takes a lock on each allocation/deallocation.
 */
class SimpleAllocator { … };

class Allocator { … };

} // namespace

void* coreMalloc(size_t size, size_t numStripes, size_t stripe) { … }

void coreFree(void* ptr) { … }

namespace {
thread_local CoreAllocatorGuard* gCoreAllocatorGuard = …;
}

CoreAllocatorGuard::CoreAllocatorGuard(size_t numStripes, size_t stripe)
    : … { … }

CoreAllocatorGuard::~CoreAllocatorGuard() { … }

namespace detail {

void* coreMallocFromGuard(size_t size) { … }

} // namespace detail

} // namespace folly