llvm/compiler-rt/lib/ctx_profile/CtxInstrProfiling.cpp

//===- CtxInstrProfiling.cpp - contextual instrumented PGO ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "CtxInstrProfiling.h"
#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_dense_map.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_mutex.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_thread_safety.h"
#include "sanitizer_common/sanitizer_vector.h"

#include <assert.h>

usingnamespace__ctx_profile;

namespace {
// Keep track of all the context roots we actually saw, so we can then traverse
// them when the user asks for the profile in __llvm_ctx_profile_fetch
__sanitizer::SpinMutex AllContextsMutex;
SANITIZER_GUARDED_BY(AllContextsMutex)
__sanitizer::Vector<ContextRoot *> AllContextRoots;

// utility to taint a pointer by setting the LSB. There is an assumption
// throughout that the addresses of contexts are even (really, they should be
// align(8), but "even"-ness is the minimum assumption)
// "scratch contexts" are buffers that we return in certain cases - they are
// large enough to allow for memory safe counter access, but they don't link
// subcontexts below them (the runtime recognizes them and enforces that)
ContextNode *markAsScratch(const ContextNode *Ctx) { … }

// Used when getting the data from TLS. We don't *really* need to reset, but
// it's a simpler system if we do.
template <typename T> inline T consume(T &V) { … }

// We allocate at least kBuffSize Arena pages. The scratch buffer is also that
// large.
constexpr size_t kPower = …;
constexpr size_t kBuffSize = …;

// Highly unlikely we need more than kBuffSize for a context.
size_t getArenaAllocSize(size_t Needed) { … }

// verify the structural integrity of the context
bool validate(const ContextRoot *Root) { … }

inline ContextNode *allocContextNode(char *Place, GUID Guid,
                                     uint32_t NrCounters, uint32_t NrCallsites,
                                     ContextNode *Next = nullptr) { … }

void resetContextNode(ContextNode &Node) { … }

void onContextEnter(ContextNode &Node) { … }

} // namespace

// the scratch buffer - what we give when we can't produce a real context (the
// scratch isn't "real" in that it's expected to be clobbered carelessly - we
// don't read it). The other important thing is that the callees from a scratch
// context also get a scratch context.
// Eventually this can be replaced with per-function buffers, a'la the typical
// (flat) instrumented FDO buffers. The clobbering aspect won't apply there, but
// the part about determining the nature of the subcontexts does.
__thread char __Buffer[kBuffSize] = …;

#define TheScratchContext …

// init the TLSes
__thread void *volatile __llvm_ctx_profile_expected_callee[2] = …;
__thread ContextNode **volatile __llvm_ctx_profile_callsite[2] = …;

__thread ContextRoot *volatile __llvm_ctx_profile_current_context_root = …;

Arena::Arena(uint32_t Size) : … { … }

// FIXME(mtrofin): use malloc / mmap instead of sanitizer common APIs to reduce
// the dependency on the latter.
Arena *Arena::allocateNewArena(size_t Size, Arena *Prev) { … }

void Arena::freeArenaList(Arena *&A) { … }

// If this is the first time we hit a callsite with this (Guid) particular
// callee, we need to allocate.
ContextNode *getCallsiteSlow(GUID Guid, ContextNode **InsertionPoint,
                             uint32_t NrCounters, uint32_t NrCallsites) { … }

ContextNode *__llvm_ctx_profile_get_context(void *Callee, GUID Guid,
                                            uint32_t NrCounters,
                                            uint32_t NrCallsites) { … }

// This should be called once for a Root. Allocate the first arena, set up the
// first context.
void setupContext(ContextRoot *Root, GUID Guid, uint32_t NrCounters,
                  uint32_t NrCallsites) { … }

ContextNode *__llvm_ctx_profile_start_context(
    ContextRoot *Root, GUID Guid, uint32_t Counters,
    uint32_t Callsites) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { … }

void __llvm_ctx_profile_release_context(ContextRoot *Root)
    SANITIZER_NO_THREAD_SAFETY_ANALYSIS { … }

void __llvm_ctx_profile_start_collection() { … }

bool __llvm_ctx_profile_fetch(void *Data,
                              bool (*Writer)(void *W, const ContextNode &)) { … }

void __llvm_ctx_profile_free() { … }