//===- MemProfiler.cpp - memory allocation and access profiler ------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of MemProfiler. Memory accesses are instrumented // to increment the access count held in a shadow memory location, or // alternatively to call into the runtime. Memory intrinsic calls (memmove, // memcpy, memset) are changed to call the memory profiling runtime version // instead. // //===----------------------------------------------------------------------===// #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/Analysis/MemoryProfileInfo.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constant.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/InstrProfReader.h" #include "llvm/Support/BLAKE3.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/HashBuilder.h" #include "llvm/Support/VirtualFileSystem.h" #include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" #include <map> #include <set> usingnamespacellvm; usingnamespacellvm::memprof; #define DEBUG_TYPE … namespace llvm { extern cl::opt<bool> PGOWarnMissing; extern cl::opt<bool> NoPGOWarnMismatch; extern cl::opt<bool> NoPGOWarnMismatchComdatWeak; } // namespace llvm constexpr int LLVM_MEM_PROFILER_VERSION = …; // Size of memory mapped to a single shadow location. constexpr uint64_t DefaultMemGranularity = …; // Size of memory mapped to a single histogram bucket. constexpr uint64_t HistogramGranularity = …; // Scale from granularity down to shadow size. constexpr uint64_t DefaultShadowScale = …; constexpr char MemProfModuleCtorName[] = …; constexpr uint64_t MemProfCtorAndDtorPriority = …; // On Emscripten, the system needs more than one priorities for constructors. constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = …; constexpr char MemProfInitName[] = …; constexpr char MemProfVersionCheckNamePrefix[] = …; constexpr char MemProfShadowMemoryDynamicAddress[] = …; constexpr char MemProfFilenameVar[] = …; constexpr char MemProfHistogramFlagVar[] = …; // Command-line flags. static cl::opt<bool> ClInsertVersionCheck( "memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true)); // This flag may need to be replaced with -f[no-]memprof-reads. static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true)); static cl::opt<bool> ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true)); static cl::opt<bool> ClInstrumentAtomics( "memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true)); static cl::opt<bool> ClUseCalls( "memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false)); static cl::opt<std::string> ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_")); // These flags allow to change the shadow mapping. // The shadow mapping looks like // Shadow = ((Mem & mask) >> scale) + offset static cl::opt<int> ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale)); static cl::opt<int> ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity)); static cl::opt<bool> ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false)); // Debug flags. static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0)); static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func")); static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1)); static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1)); // By default disable matching of allocation profiles onto operator new that // already explicitly pass a hot/cold hint, since we don't currently // override these hints anyway. static cl::opt<bool> ClMemProfMatchHotColdNew( "memprof-match-hot-cold-new", cl::desc( "Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false)); static cl::opt<bool> ClHistogram("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false)); static cl::opt<bool> ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false)); extern cl::opt<bool> MemProfReportHintedSizes; // Instrumentation statistics STATISTIC(NumInstrumentedReads, "Number of instrumented reads"); STATISTIC(NumInstrumentedWrites, "Number of instrumented writes"); STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads"); STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes"); // Matching statistics STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile."); STATISTIC(NumOfMemProfMismatch, "Number of functions having mismatched memory profile hash."); STATISTIC(NumOfMemProfFunc, "Number of functions having valid memory profile."); STATISTIC(NumOfMemProfAllocContextProfiles, "Number of alloc contexts in memory profile."); STATISTIC(NumOfMemProfCallSiteProfiles, "Number of callsites in memory profile."); STATISTIC(NumOfMemProfMatchedAllocContexts, "Number of matched memory profile alloc contexts."); STATISTIC(NumOfMemProfMatchedAllocs, "Number of matched memory profile allocs."); STATISTIC(NumOfMemProfMatchedCallSites, "Number of matched memory profile callsites."); namespace { /// This struct defines the shadow mapping using the rule: /// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset. struct ShadowMapping { … }; static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) { … } struct InterestingMemoryAccess { … }; /// Instrument the code in module to profile memory accesses. class MemProfiler { … }; class ModuleMemProfiler { … }; } // end anonymous namespace MemProfilerPass::MemProfilerPass() = default; PreservedAnalyses MemProfilerPass::run(Function &F, AnalysisManager<Function> &AM) { … } ModuleMemProfilerPass::ModuleMemProfilerPass() = default; PreservedAnalyses ModuleMemProfilerPass::run(Module &M, AnalysisManager<Module> &AM) { … } Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) { … } // Instrument memset/memmove/memcpy void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) { … } std::optional<InterestingMemoryAccess> MemProfiler::isInterestingMemoryAccess(Instruction *I) const { … } void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask, Instruction *I, Value *Addr, Type *AccessTy, bool IsWrite) { … } void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL, InterestingMemoryAccess &Access) { … } void MemProfiler::instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, bool IsWrite) { … } // Create the variable for the profile file name. void createProfileFileNameVar(Module &M) { … } // Set MemprofHistogramFlag as a Global veriable in IR. This makes it accessible // to the runtime, changing shadow count behavior. void createMemprofHistogramFlagVar(Module &M) { … } bool ModuleMemProfiler::instrumentModule(Module &M) { … } void MemProfiler::initializeCallbacks(Module &M) { … } bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) { … } bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) { … } bool MemProfiler::instrumentFunction(Function &F) { … } static void addCallsiteMetadata(Instruction &I, std::vector<uint64_t> &InlinedCallStack, LLVMContext &Ctx) { … } static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column) { … } static uint64_t computeStackId(const memprof::Frame &Frame) { … } // Helper to generate a single hash id for a given callstack, used for emitting // matching statistics and useful for uniquing such statistics across modules. static uint64_t computeFullStackId(const std::vector<memprof::Frame> &CallStack) { … } static AllocationType addCallStack(CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo) { … } // Helper to compare the InlinedCallStack computed from an instruction's debug // info to a list of Frames from profile data (either the allocation data or a // callsite). For callsites, the StartIndex to use in the Frame array may be // non-zero. static bool stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack, ArrayRef<uint64_t> InlinedCallStack, unsigned StartIndex = 0) { … } static bool isAllocationWithHotColdVariant(Function *Callee, const TargetLibraryInfo &TLI) { … } struct AllocMatchInfo { … }; static void readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map<uint64_t, AllocMatchInfo> &FullStackIdToAllocMatchInfo) { … } MemProfUsePass::MemProfUsePass(std::string MemoryProfileFile, IntrusiveRefCntPtr<vfs::FileSystem> FS) : … { … } PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) { … }
Codebase Browser
llvm