llvm/llvm/include/llvm/ProfileData/InstrProf.h

//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Instrumentation-based profiling data is generated by instrumented
// binaries through library functions in compiler-rt, and read by the clang
// frontend to feed PGO.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_PROFILEDATA_INSTRPROF_H
#define LLVM_PROFILEDATA_INSTRPROF_H

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/IntervalMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/ProfileData/InstrProfData.inc"
#include "llvm/Support/BalancedPartitioning.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MD5.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Host.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <list>
#include <memory>
#include <string>
#include <system_error>
#include <utility>
#include <vector>

namespace llvm {

class Function;
class GlobalVariable;
struct InstrProfRecord;
class InstrProfSymtab;
class Instruction;
class MDNode;
class Module;

enum InstrProfSectKind { … };

/// Return the max count value. We reserver a few large values for special use.
inline uint64_t getInstrMaxCountValue() { … }

/// Return the name of the profile section corresponding to \p IPSK.
///
/// The name of the section depends on the object format type \p OF. If
/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
/// be added to the section name (this is the default).
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
                                    Triple::ObjectFormatType OF,
                                    bool AddSegmentInfo = true);

/// Return the name profile runtime entry point to do value profiling
/// for a given site.
inline StringRef getInstrProfValueProfFuncName() { … }

/// Return the name profile runtime entry point to do memop size value
/// profiling.
inline StringRef getInstrProfValueProfMemOpFuncName() { … }

/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { … }

/// Return the name prefix of variables containing virtual table profile data.
inline StringRef getInstrProfVTableVarPrefix() { … }

/// Return the name prefix of variables containing per-function control data.
inline StringRef getInstrProfDataVarPrefix() { … }

/// Return the name prefix of profile counter variables.
inline StringRef getInstrProfCountersVarPrefix() { … }

/// Return the name prefix of profile bitmap variables.
inline StringRef getInstrProfBitmapVarPrefix() { … }

/// Return the name prefix of value profile variables.
inline StringRef getInstrProfValuesVarPrefix() { … }

/// Return the name of value profile node array variables:
inline StringRef getInstrProfVNodesVarName() { … }

/// Return the name of the variable holding the strings (possibly compressed)
/// of all function's PGO names.
inline StringRef getInstrProfNamesVarName() { … }

inline StringRef getInstrProfVTableNamesVarName() { … }

/// Return the name of a covarage mapping variable (internal linkage)
/// for each instrumented source module. Such variables are allocated
/// in the __llvm_covmap section.
inline StringRef getCoverageMappingVarName() { … }

/// Return the name of the internal variable recording the array
/// of PGO name vars referenced by the coverage mapping. The owning
/// functions of those names are not emitted by FE (e.g, unused inline
/// functions.)
inline StringRef getCoverageUnusedNamesVarName() { … }

/// Return the name of function that registers all the per-function control
/// data at program startup time by calling __llvm_register_function. This
/// function has internal linkage and is called by  __llvm_profile_init
/// runtime method. This function is not generated for these platforms:
/// Darwin, Linux, and FreeBSD.
inline StringRef getInstrProfRegFuncsName() { … }

/// Return the name of the runtime interface that registers per-function control
/// data for one instrumented function.
inline StringRef getInstrProfRegFuncName() { … }

/// Return the name of the runtime interface that registers the PGO name
/// strings.
inline StringRef getInstrProfNamesRegFuncName() { … }

/// Return the name of the runtime initialization method that is generated by
/// the compiler. The function calls __llvm_profile_register_functions and
/// __llvm_profile_override_default_filename functions if needed. This function
/// has internal linkage and invoked at startup time via init_array.
inline StringRef getInstrProfInitFuncName() { … }

/// Return the name of the hook variable defined in profile runtime library.
/// A reference to the variable causes the linker to link in the runtime
/// initialization module (which defines the hook variable).
inline StringRef getInstrProfRuntimeHookVarName() { … }

/// Return the name of the compiler generated function that references the
/// runtime hook variable. The function is a weak global.
inline StringRef getInstrProfRuntimeHookVarUseFuncName() { … }

inline StringRef getInstrProfCounterBiasVarName() { … }

inline StringRef getInstrProfBitmapBiasVarName() { … }

/// Return the marker used to separate PGO names during serialization.
inline StringRef getInstrProfNameSeparator() { … }

/// Determines whether module targets a GPU eligable for PGO
/// instrumentation
bool isGPUProfTarget(const Module &M);

/// Please use getIRPGOFuncName for LLVM IR instrumentation. This function is
/// for front-end (Clang, etc) instrumentation.
/// Return the modified name for function \c F suitable to be
/// used the key for profile lookup. Variable \c InLTO indicates if this
/// is called in LTO optimization passes.
std::string getPGOFuncName(const Function &F, bool InLTO = false,
                           uint64_t Version = INSTR_PROF_INDEX_VERSION);

/// Return the modified name for a function suitable to be
/// used the key for profile lookup. The function's original
/// name is \c RawFuncName and has linkage of type \c Linkage.
/// The function is defined in module \c FileName.
std::string getPGOFuncName(StringRef RawFuncName,
                           GlobalValue::LinkageTypes Linkage,
                           StringRef FileName,
                           uint64_t Version = INSTR_PROF_INDEX_VERSION);

/// \return the modified name for function \c F suitable to be
/// used as the key for IRPGO profile lookup. \c InLTO indicates if this is
/// called from LTO optimization passes.
std::string getIRPGOFuncName(const Function &F, bool InLTO = false);

/// \return the filename and the function name parsed from the output of
/// \c getIRPGOFuncName()
std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName);

/// Return the name of the global variable used to store a function
/// name in PGO instrumentation. \c FuncName is the IRPGO function name
/// (returned by \c getIRPGOFuncName) for LLVM IR instrumentation and PGO
/// function name (returned by \c getPGOFuncName) for front-end instrumentation.
std::string getPGOFuncNameVarName(StringRef FuncName,
                                  GlobalValue::LinkageTypes Linkage);

/// Create and return the global variable for function name used in PGO
/// instrumentation. \c FuncName is the IRPGO function name (returned by
/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
/// (returned by \c getPGOFuncName) for front-end instrumentation.
GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);

/// Create and return the global variable for function name used in PGO
/// instrumentation. \c FuncName is the IRPGO function name (returned by
/// \c getIRPGOFuncName) for LLVM IR instrumentation and PGO function name
/// (returned by \c getPGOFuncName) for front-end instrumentation.
GlobalVariable *createPGOFuncNameVar(Module &M,
                                     GlobalValue::LinkageTypes Linkage,
                                     StringRef PGOFuncName);

/// Return the initializer in string of the PGO name var \c NameVar.
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);

/// Given a PGO function name, remove the filename prefix and return
/// the original (static) function name.
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
                                   StringRef FileName = "<unknown>");

/// Given a vector of strings (names of global objects like functions or,
/// virtual tables) \c NameStrs, the method generates a combined string \c
/// Result that is ready to be serialized.  The \c Result string is comprised of
/// three fields: The first field is the length of the uncompressed strings, and
/// the the second field is the length of the zlib-compressed string. Both
/// fields are encoded in ULEB128.  If \c doCompress is false, the
///  third field is the uncompressed strings; otherwise it is the
/// compressed string. When the string compression is off, the
/// second field will have value zero.
Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs,
                                     bool doCompression, std::string &Result);

/// Produce \c Result string with the same format described above. The input
/// is vector of PGO function name variables that are referenced.
/// The global variable element in 'NameVars' is a string containing the pgo
/// name of a function. See `createPGOFuncNameVar` that creates these global
/// variables.
Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
                                std::string &Result, bool doCompression = true);

Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables,
                           std::string &Result, bool doCompression);

/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
/// set in IR PGO compilation.
bool isIRPGOFlagSet(const Module *M);

/// Check if we can safely rename this Comdat function. Instances of the same
/// comdat function may have different control flows thus can not share the
/// same counter variable.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);

enum InstrProfValueKind : uint32_t { … };

/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
/// and annotate the instruction \p Inst with the value profile meta data.
/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
void annotateValueSite(Module &M, Instruction &Inst,
                       const InstrProfRecord &InstrProfR,
                       InstrProfValueKind ValueKind, uint32_t SiteIndx,
                       uint32_t MaxMDCount = 3);

/// Same as the above interface but using an ArrayRef, as well as \p Sum.
/// This function will not annotate !prof metadata on the instruction if the
/// referenced array is empty.
void annotateValueSite(Module &M, Instruction &Inst,
                       ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
                       InstrProfValueKind ValueKind, uint32_t MaxMDCount);

// TODO: Unify metadata name 'PGOFuncName' and 'PGOName', by supporting read
// of this metadata for backward compatibility and generating 'PGOName' only.
/// Extract the value profile data from \p Inst and returns them if \p Inst is
/// annotated with value profile data. Returns an empty vector otherwise.
SmallVector<InstrProfValueData, 4>
getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind,
                         uint32_t MaxNumValueData, uint64_t &TotalC,
                         bool GetNoICPValue = false);

inline StringRef getPGOFuncNameMetadataName() { … }

inline StringRef getPGONameMetadataName() { … }

/// Return the PGOFuncName meta data associated with a function.
MDNode *getPGOFuncNameMetadata(const Function &F);

std::string getPGOName(const GlobalVariable &V, bool InLTO = false);

/// Create the PGOFuncName meta data if PGOFuncName is different from
/// function's raw name. This should only apply to internal linkage functions
/// declared by users only.
/// TODO: Update all callers to 'createPGONameMetadata' and deprecate this
/// function.
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);

/// Create the PGOName metadata if a global object's PGO name is different from
/// its mangled name. This should apply to local-linkage global objects only.
void createPGONameMetadata(GlobalObject &GO, StringRef PGOName);

/// Check if we can use Comdat for profile variables. This will eliminate
/// the duplicated profile variables for Comdat functions.
bool needsComdatForCounter(const GlobalObject &GV, const Module &M);

/// An enum describing the attributes of an instrumented profile.
enum class InstrProfKind { … };

const std::error_category &instrprof_category();

enum class instrprof_error { … };

/// An ordered list of functions identified by their NameRef found in
/// INSTR_PROF_DATA
struct TemporalProfTraceTy { … };

inline std::error_code make_error_code(instrprof_error E) { … }

class InstrProfError : public ErrorInfo<InstrProfError> { … };

namespace object {

class SectionRef;

} // end namespace object

namespace IndexedInstrProf {

uint64_t ComputeHash(StringRef K);

} // end namespace IndexedInstrProf

/// A symbol table used for function [IR]PGO name look-up with keys
/// (such as pointers, md5hash values) to the function. A function's
/// [IR]PGO name or name's md5hash are used in retrieving the profile
/// data of the function. See \c getIRPGOFuncName() and \c getPGOFuncName
/// methods for details how [IR]PGO name is formed.
class InstrProfSymtab { … };

Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { … }

template <typename NameIterRange>
Error InstrProfSymtab::create(const NameIterRange &IterRange) { … }

template <typename FuncNameIterRange, typename VTableNameIterRange>
Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
                              const VTableNameIterRange &VTableIterRange) { … }

void InstrProfSymtab::finalizeSymtab() { … }

StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) { … }

StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) { … }

Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { … }

GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) { … }

// To store the sums of profile count values, or the percentage of
// the sums of the total count values.
struct CountSumOrPercent { … };

// Function level or program level overlap information.
struct OverlapStats { … };

// This is used to filter the functions whose overlap information
// to be output.
struct OverlapFuncFilters { … };

struct InstrProfValueSiteRecord { … };

/// Profiling information for a single function.
struct InstrProfRecord { … };

struct NamedInstrProfRecord : InstrProfRecord { … };

uint32_t InstrProfRecord::getNumValueKinds() const { … }

uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { … }

uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { … }

ArrayRef<InstrProfValueData>
InstrProfRecord::getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const { … }

void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { … }

// Include definitions for value profile data
#define INSTR_PROF_VALUE_PROF_DATA
#include "llvm/ProfileData/InstrProfData.inc"

void InstrProfValueSiteRecord::sortByCount() { … }

namespace IndexedInstrProf {

enum class HashT : uint32_t { … };

inline uint64_t ComputeHash(HashT Type, StringRef K) { … }

const uint64_t Magic = …; // "\xfflprofi\x81"

enum ProfVersion { … };
const uint64_t Version = …;

const HashT HashType = …;

inline uint64_t ComputeHash(StringRef K) { … }

// This structure defines the file header of the LLVM profile
// data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst
// as appropriate when updating the indexed profile format.
struct Header { … };

// Profile summary data recorded in the profile data file in indexed
// format. It is introduced in version 4. The summary data follows
// right after the profile file header.
struct Summary { … };

inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) { … }

} // end namespace IndexedInstrProf

namespace RawInstrProf {

// Version 1: First version
// Version 2: Added value profile data section. Per-function control data
// struct has more fields to describe value profile information.
// Version 3: Compressed name section support. Function PGO name reference
// from control data struct is changed from raw pointer to Name's MD5 value.
// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
// raw header.
// Version 5: Bit 60 of FuncHash is reserved for the flag for the context
// sensitive records.
// Version 6: Added binary id.
// Version 7: Reorder binary id and include version in signature.
// Version 8: Use relative counter pointer.
// Version 9: Added relative bitmap bytes pointer and count used by MC/DC.
// Version 10: Added vtable, a new type of value profile data.
const uint64_t Version = …;

template <class IntPtrT> inline uint64_t getMagic();
template <> inline uint64_t getMagic<uint64_t>() { … }

template <> inline uint64_t getMagic<uint32_t>() { … }

// Per-function profile data header/control structure.
// The definition should match the structure defined in
// compiler-rt/lib/profile/InstrProfiling.h.
// It should also match the synthesized type in
// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
template <class IntPtrT> struct alignas(8) ProfileData { … };

template <class IntPtrT> struct alignas(8) VTableProfileData { … };

// File header structure of the LLVM profile data in raw format.
// The definition should match the header referenced in
// compiler-rt/lib/profile/InstrProfilingFile.c  and
// InstrProfilingBuffer.c.
struct Header { … };

} // end namespace RawInstrProf

// Create the variable for the profile file name.
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);

// Whether to compress function names in profile records, and filenames in
// code coverage mappings. Used by the Instrumentation library and unit tests.
extern cl::opt<bool> DoInstrProfNameCompression;

} // end namespace llvm
#endif // LLVM_PROFILEDATA_INSTRPROF_H