//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This pass lowers instrprof_* intrinsics emitted by an instrumentor. // It also builds the data structures and initialization code needed for // updating execution counts and emitting the profile at runtime. // //===----------------------------------------------------------------------===// #include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfCorrelator.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Instrumentation.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include "llvm/Transforms/Utils/SSAUpdater.h" #include <algorithm> #include <cassert> #include <cstdint> #include <string> usingnamespacellvm; #define DEBUG_TYPE … namespace llvm { // Command line option to enable vtable value profiling. Defined in // ProfileData/InstrProf.cpp: -enable-vtable-value-profiling= extern cl::opt<bool> EnableVTableValueProfiling; // TODO: Remove -debug-info-correlate in next LLVM release, in favor of // -profile-correlate=debug-info. cl::opt<bool> DebugInfoCorrelate( "debug-info-correlate", cl::desc("Use debug info to correlate profiles. (Deprecated, use " "-profile-correlate=debug-info)"), cl::init(false)); cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate( "profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate"))); } // namespace llvm namespace { cl::opt<bool> DoHashBasedCounterSplit( "hash-based-counter-split", cl::desc("Rename counter variable of a comdat function based on cfg hash"), cl::init(true)); cl::opt<bool> RuntimeCounterRelocation("runtime-counter-relocation", cl::desc("Enable relocating counters at runtime."), cl::init(false)); cl::opt<bool> ValueProfileStaticAlloc( "vp-static-alloc", cl::desc("Do static counter allocation for value profiler"), cl::init(true)); cl::opt<double> NumCountersPerValueSite( "vp-counters-per-site", cl::desc("The average number of profile counters allocated " "per value profiling site."), // This is set to a very small value because in real programs, only // a very small percentage of value sites have non-zero targets, e.g, 1/30. // For those sites with non-zero profile, the average number of targets // is usually smaller than 2. cl::init(1.0)); cl::opt<bool> AtomicCounterUpdateAll( "instrprof-atomic-counter-update-all", cl::desc("Make all profile counter updates atomic (for testing only)"), cl::init(false)); cl::opt<bool> AtomicCounterUpdatePromoted( "atomic-counter-update-promoted", cl::desc("Do counter update using atomic fetch add " " for promoted counters only"), cl::init(false)); cl::opt<bool> AtomicFirstCounter( "atomic-first-counter", cl::desc("Use atomic fetch add for first counter in a function (usually " "the entry counter)"), cl::init(false)); cl::opt<bool> ConditionalCounterUpdate( "conditional-counter-update", cl::desc("Do conditional counter updates in single byte counters mode)"), cl::init(false)); // If the option is not specified, the default behavior about whether // counter promotion is done depends on how instrumentaiton lowering // pipeline is setup, i.e., the default value of true of this option // does not mean the promotion will be done by default. Explicitly // setting this option can override the default behavior. cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::desc("Do counter register promotion"), cl::init(false)); cl::opt<unsigned> MaxNumOfPromotionsPerLoop( "max-counter-promotions-per-loop", cl::init(20), cl::desc("Max number counter promotions per loop to avoid" " increasing register pressure too much")); // A debug option cl::opt<int> MaxNumOfPromotions("max-counter-promotions", cl::init(-1), cl::desc("Max number of allowed counter promotions")); cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting( "speculative-counter-promotion-max-exiting", cl::init(3), cl::desc("The max number of exiting blocks of a loop to allow " " speculative counter promotion")); cl::opt<bool> SpeculativeCounterPromotionToLoop( "speculative-counter-promotion-to-loop", cl::desc("When the option is false, if the target block is in a loop, " "the promotion will be disallowed unless the promoted counter " " update can be further/iteratively promoted into an acyclic " " region.")); cl::opt<bool> IterativeCounterPromotion( "iterative-counter-promotion", cl::init(true), cl::desc("Allow counter promotion across the whole loop nest.")); cl::opt<bool> SkipRetExitBlock( "skip-ret-exit-block", cl::init(true), cl::desc("Suppress counter promotion if exit blocks contain ret.")); static cl::opt<bool> SampledInstr("sampled-instrumentation", cl::ZeroOrMore, cl::init(false), cl::desc("Do PGO instrumentation sampling")); static cl::opt<unsigned> SampledInstrPeriod( "sampled-instr-period", cl::desc("Set the profile instrumentation sample period. For each sample " "period, a fixed number of consecutive samples will be recorded. " "The number is controlled by 'sampled-instr-burst-duration' flag. " "The default sample period of 65535 is optimized for generating " "efficient code that leverages unsigned integer wrapping in " "overflow."), cl::init(65535)); static cl::opt<unsigned> SampledInstrBurstDuration( "sampled-instr-burst-duration", cl::desc("Set the profile instrumentation burst duration, which can range " "from 0 to one less than the value of 'sampled-instr-period'. " "This number of samples will be recorded for each " "'sampled-instr-period' count update. Setting to 1 enables " "simple sampling, in which case it is recommended to set " "'sampled-instr-period' to a prime number."), cl::init(200)); LoadStorePair; static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) { … } static bool enablesValueProfiling(const Module &M) { … } // Conservatively returns true if value profiling is enabled. static bool profDataReferencedByCode(const Module &M) { … } class InstrLowerer final { … }; /// /// A helper class to promote one counter RMW operation in the loop /// into register update. /// /// RWM update for the counter will be sinked out of the loop after /// the transformation. /// class PGOCounterPromoterHelper : public LoadAndStorePromoter { … }; /// A helper class to do register promotion for all profile counter /// updates in a loop. /// class PGOCounterPromoter { … }; enum class ValueProfilingCallType { … }; } // end anonymous namespace PreservedAnalyses InstrProfilingLoweringPass::run(Module &M, ModuleAnalysisManager &AM) { … } // // Perform instrumentation sampling. // // There are 3 favors of sampling: // (1) Full burst sampling: We transform: // Increment_Instruction; // to: // if (__llvm_profile_sampling__ < SampledInstrBurstDuration) { // Increment_Instruction; // } // __llvm_profile_sampling__ += 1; // if (__llvm_profile_sampling__ >= SampledInstrPeriod) { // __llvm_profile_sampling__ = 0; // } // // "__llvm_profile_sampling__" is a thread-local global shared by all PGO // counters (value-instrumentation and edge instrumentation). // // (2) Fast burst sampling: // "__llvm_profile_sampling__" variable is an unsigned type, meaning it will // wrap around to zero when overflows. In this case, the second check is // unnecessary, so we won't generate check2 when the SampledInstrPeriod is // set to 65535 (64K - 1). The code after: // if (__llvm_profile_sampling__ < SampledInstrBurstDuration) { // Increment_Instruction; // } // __llvm_profile_sampling__ += 1; // // (3) Simple sampling: // When SampledInstrBurstDuration sets to 1, we do a simple sampling: // __llvm_profile_sampling__ += 1; // if (__llvm_profile_sampling__ >= SampledInstrPeriod) { // __llvm_profile_sampling__ = 0; // Increment_Instruction; // } // // Note that, the code snippet after the transformation can still be counter // promoted. However, with sampling enabled, counter updates are expected to // be infrequent, making the benefits of counter promotion negligible. // Moreover, counter promotion can potentially cause issues in server // applications, particularly when the counters are dumped without a clean // exit. To mitigate this risk, counter promotion is disabled by default when // sampling is enabled. This behavior can be overridden using the internal // option. void InstrLowerer::doSampling(Instruction *I) { … } bool InstrLowerer::lowerIntrinsics(Function *F) { … } bool InstrLowerer::isRuntimeCounterRelocationEnabled() const { … } bool InstrLowerer::isSamplingEnabled() const { … } bool InstrLowerer::isCounterPromotionEnabled() const { … } void InstrLowerer::promoteCounterLoadStores(Function *F) { … } static bool needsRuntimeHookUnconditionally(const Triple &TT) { … } /// Check if the module contains uses of any profiling intrinsics. static bool containsProfilingIntrinsics(Module &M) { … } bool InstrLowerer::lower() { … } static FunctionCallee getOrInsertValueProfilingCall( Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType = ValueProfilingCallType::Default) { … } void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) { … } void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { … } GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) { … } Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) { … } /// Create `void [[alwaysinline]] rmw_or(uint8_t *ArgAddr, uint8_t ArgVal)` /// "Basic" sequence is `*ArgAddr |= ArgVal` Function *InstrLowerer::createRMWOrFunc() { … } CallInst *InstrLowerer::getRMWOrCall(Value *Addr, Value *Val) { … } Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { … } void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) { … } void InstrLowerer::lowerTimestamp( InstrProfTimestampInst *TimestampInstruction) { … } void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) { … } void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) { … } void InstrLowerer::lowerMCDCTestVectorBitmapUpdate( InstrProfMCDCTVBitmapUpdate *Update) { … } /// Get the name of a profiling variable for a particular function. static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed) { … } static inline bool shouldRecordFunctionAddr(Function *F) { … } static inline bool shouldUsePublicSymbol(Function *Fn) { … } static inline Constant *getFuncAddrForProfData(Function *Fn) { … } static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { … } void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef CounterGroupName) { … } static inline bool shouldRecordVTableAddr(GlobalVariable *GV) { … } // FIXME: Introduce an internal alias like what's done for functions to reduce // the number of relocation entries. static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) { … } void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) { … } GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc, InstrProfSectKind IPSK) { … } GlobalVariable * InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage) { … } GlobalVariable * InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) { … } GlobalVariable * InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage) { … } GlobalVariable * InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { … } void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) { … } void InstrLowerer::emitVNodes() { … } void InstrLowerer::emitNameData() { … } void InstrLowerer::emitVTableNames() { … } void InstrLowerer::emitRegistration() { … } bool InstrLowerer::emitRuntimeHook() { … } void InstrLowerer::emitUses() { … } void InstrLowerer::emitInitialization() { … } namespace llvm { // Create the variable for profile sampling. void createProfileSamplingVar(Module &M) { … } } // namespace llvm
Codebase Browser
llvm