//===- InstrProf.cpp - Instrumented profiling format support --------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains support for clang's instrumentation based PGO and // coverage. // //===----------------------------------------------------------------------===// #include "llvm/ProfileData/InstrProf.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/Config/config.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Compression.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/LEB128.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Path.h" #include "llvm/Support/SwapByteOrder.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/TargetParser/Triple.h" #include <algorithm> #include <cassert> #include <cstddef> #include <cstdint> #include <cstring> #include <memory> #include <string> #include <system_error> #include <type_traits> #include <utility> #include <vector> usingnamespacellvm; #define DEBUG_TYPE … static cl::opt<bool> StaticFuncFullModulePrefix( "static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions.")); // This option is tailored to users that have different top-level directory in // profile-gen and profile-use compilation. Users need to specific the number // of levels to strip. A value larger than the number of directories in the // source file will strip all the directory names and only leave the basename. // // Note current ThinLTO module importing for the indirect-calls assumes // the source directory name not being stripped. A non-zero option value here // can potentially prevent some inter-module indirect-call-promotions. static cl::opt<unsigned> StaticFuncStripDirNamePrefix( "static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions.")); static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg = "") { … } namespace { // FIXME: This class is only here to support the transition to llvm::Error. It // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. class InstrProfErrorCategoryType : public std::error_category { … }; } // end anonymous namespace const std::error_category &llvm::instrprof_category() { … } namespace { const char *InstrProfSectNameCommon[] = …; const char *InstrProfSectNameCoff[] = …; const char *InstrProfSectNamePrefix[] = …; } // namespace namespace llvm { cl::opt<bool> DoInstrProfNameCompression( "enable-name-compression", cl::desc("Enable name/filename string compression"), cl::init(true)); cl::opt<bool> EnableVTableValueProfiling( "enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison.")); cl::opt<bool> EnableVTableProfileUse( "enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used.")); std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo) { … } std::string InstrProfError::message() const { … } char InstrProfError::ID = …; std::string getPGOFuncName(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName, uint64_t Version LLVM_ATTRIBUTE_UNUSED) { … } // Strip NumPrefix level of directory name from PathNameStr. If the number of // directory separators is less than NumPrefix, strip all the directories and // leave base file name only. static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) { … } static StringRef getStrippedSourceFileName(const GlobalObject &GO) { … } // The PGO name has the format [<filepath>;]<mangled-name> where <filepath>; is // provided if linkage is local and is used to discriminate possibly identical // mangled names. ";" is used because it is unlikely to be found in either // <filepath> or <mangled-name>. // // Older compilers used getPGOFuncName() which has the format // [<filepath>:]<mangled-name>. This caused trouble for Objective-C functions // which commonly have :'s in their names. We still need to compute this name to // lookup functions from profiles built by older compilers. static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName) { … } static std::optional<std::string> lookupPGONameFromMetadata(MDNode *MD) { … } // Returns the PGO object name. This function has some special handling // when called in LTO optimization. The following only applies when calling in // LTO passes (when \c InLTO is true): LTO's internalization privatizes many // global linkage symbols. This happens after value profile annotation, but // those internal linkage functions should not have a source prefix. // Additionally, for ThinLTO mode, exported internal functions are promoted // and renamed. We need to ensure that the original internal PGO name is // used when computing the GUID that is compared against the profiled GUIDs. // To differentiate compiler generated internal symbols from original ones, // PGOFuncName meta data are created and attached to the original internal // symbols in the value profile annotation step // (PGOUseFunc::annotateIndirectCallSites). If a symbol does not have the meta // data, its original linkage must be non-internal. static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata) { … } // Returns the IRPGO function name and does special handling when called // in LTO optimization. See the comments of `getIRPGOObjectName` for details. std::string getIRPGOFuncName(const Function &F, bool InLTO) { … } // Please use getIRPGOFuncName for LLVM IR instrumentation. This function is // for front-end (Clang, etc) instrumentation. // The implementation is kept for profile matching from older profiles. // This is similar to `getIRPGOFuncName` except that this function calls // 'getPGOFuncName' to get a name and `getIRPGOFuncName` calls // 'getIRPGONameForGlobalObject'. See the difference between two callees in the // comments of `getIRPGONameForGlobalObject`. std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { … } std::string getPGOName(const GlobalVariable &V, bool InLTO) { … } // See getIRPGOObjectName() for a discription of the format. std::pair<StringRef, StringRef> getParsedIRPGOName(StringRef IRPGOName) { … } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { … } // \p FuncName is the string used as profile lookup key for the function. A // symbol is created to hold the name. Return the legalized symbol name. std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage) { … } bool isGPUProfTarget(const Module &M) { … } void setPGOFuncVisibility(Module &M, GlobalVariable *FuncNameVar) { … } GlobalVariable *createPGOFuncNameVar(Module &M, GlobalValue::LinkageTypes Linkage, StringRef PGOFuncName) { … } GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName) { … } Error InstrProfSymtab::create(Module &M, bool InLTO) { … } Error InstrProfSymtab::addVTableWithName(GlobalVariable &VTable, StringRef VTablePGOName) { … } /// \c NameStrings is a string composed of one of more possibly encoded /// sub-strings. The substrings are separated by 0 or more zero bytes. This /// method decodes the string and calls `NameCallback` for each substring. static Error readAndDecodeStrings(StringRef NameStrings, std::function<Error(StringRef)> NameCallback) { … } Error InstrProfSymtab::create(StringRef NameStrings) { … } Error InstrProfSymtab::create(StringRef FuncNameStrings, StringRef VTableNameStrings) { … } Error InstrProfSymtab::initVTableNamesFromCompressedStrings( StringRef CompressedVTableStrings) { … } StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) { … } Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { … } uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) { … } uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { … } void InstrProfSymtab::dumpNames(raw_ostream &OS) const { … } Error collectGlobalObjectNameStrings(ArrayRef<std::string> NameStrs, bool DoCompression, std::string &Result) { … } StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar) { … } Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, std::string &Result, bool DoCompression) { … } Error collectVTableStrings(ArrayRef<GlobalVariable *> VTables, std::string &Result, bool DoCompression) { … } void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { … } void InstrProfValueSiteRecord::overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap) { … } // Return false on mismatch. void InstrProfRecord::overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap) { … } void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff) { … } void InstrProfValueSiteRecord::merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref<void(instrprof_error)> Warn) { … } void InstrProfValueSiteRecord::scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn) { … } // Merge Value Profile data from Src record to this record for ValueKind. // Scale merged value counts by \p Weight. void InstrProfRecord::mergeValueProfData( uint32_t ValueKind, InstrProfRecord &Src, uint64_t Weight, function_ref<void(instrprof_error)> Warn) { … } void InstrProfRecord::merge(InstrProfRecord &Other, uint64_t Weight, function_ref<void(instrprof_error)> Warn) { … } void InstrProfRecord::scaleValueProfData( uint32_t ValueKind, uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn) { … } void InstrProfRecord::scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn) { … } // Map indirect call target name hash to name string. uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, InstrProfSymtab *SymTab) { … } void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site, ArrayRef<InstrProfValueData> VData, InstrProfSymtab *ValueMap) { … } void TemporalProfTraceTy::createBPFunctionNodes( ArrayRef<TemporalProfTraceTy> Traces, std::vector<BPFunctionNode> &Nodes, bool RemoveOutlierUNs) { … } #define INSTR_PROF_COMMON_API_IMPL #include "llvm/ProfileData/InstrProfData.inc" /*! * ValueProfRecordClosure Interface implementation for InstrProfRecord * class. These C wrappers are used as adaptors so that C++ code can be * invoked as callbacks. */ uint32_t getNumValueKindsInstrProf(const void *Record) { … } uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { … } uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { … } uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S) { … } void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S) { … } ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { … } static ValueProfRecordClosure InstrProfRecordClosure = …; // Wrapper implementation using the closure mechanism. uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { … } // Wrapper implementation using the closure mechanism. std::unique_ptr<ValueProfData> ValueProfData::serializeFrom(const InstrProfRecord &Record) { … } void ValueProfRecord::deserializeTo(InstrProfRecord &Record, InstrProfSymtab *SymTab) { … } // For writing/serializing, Old is the host endianness, and New is // byte order intended on disk. For Reading/deserialization, Old // is the on-disk source endianness, and New is the host endianness. void ValueProfRecord::swapBytes(llvm::endianness Old, llvm::endianness New) { … } void ValueProfData::deserializeTo(InstrProfRecord &Record, InstrProfSymtab *SymTab) { … } static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) { … } Error ValueProfData::checkIntegrity() { … } Expected<std::unique_ptr<ValueProfData>> ValueProfData::getValueProfData(const unsigned char *D, const unsigned char *const BufferEnd, llvm::endianness Endianness) { … } void ValueProfData::swapBytesToHost(llvm::endianness Endianness) { … } void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) { … } void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIdx, uint32_t MaxMDCount) { … } void annotateValueSite(Module &M, Instruction &Inst, ArrayRef<InstrProfValueData> VDs, uint64_t Sum, InstrProfValueKind ValueKind, uint32_t MaxMDCount) { … } MDNode *mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind) { … } SmallVector<InstrProfValueData, 4> getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue) { … } MDNode *getPGOFuncNameMetadata(const Function &F) { … } static void createPGONameMetadata(GlobalObject &GO, StringRef MetadataName, StringRef PGOName) { … } void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { … } void createPGONameMetadata(GlobalObject &GO, StringRef PGOName) { … } bool needsComdatForCounter(const GlobalObject &GO, const Module &M) { … } // Check if INSTR_PROF_RAW_VERSION_VAR is defined. bool isIRPGOFlagSet(const Module *M) { … } // Check if we can safely rename this Comdat function. bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) { … } // Create the variable for the profile file name. void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) { … } Error OverlapStats::accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS) { … } void OverlapStats::addOneMismatch(const CountSumOrPercent &MismatchFunc) { … } void OverlapStats::addOneUnique(const CountSumOrPercent &UniqueFunc) { … } void OverlapStats::dump(raw_fd_ostream &OS) const { … } namespace IndexedInstrProf { Expected<Header> Header::readFromBuffer(const unsigned char *Buffer) { … } uint64_t Header::getIndexedProfileVersion() const { … } size_t Header::size() const { … } } // namespace IndexedInstrProf } // end namespace llvm
Codebase Browser
llvm