//===- bolt/Passes/IndirectCallPromotion.cpp ------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the IndirectCallPromotion class. // //===----------------------------------------------------------------------===// #include "bolt/Passes/IndirectCallPromotion.h" #include "bolt/Core/BinaryFunctionCallGraph.h" #include "bolt/Passes/DataflowInfoManager.h" #include "bolt/Passes/Inliner.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/CommandLine.h" #include <iterator> #define DEBUG_TYPE … #define DEBUG_VERBOSE(Level, X) … usingnamespacellvm; usingnamespacebolt; namespace opts { extern cl::OptionCategory BoltOptCategory; extern cl::opt<IndirectCallPromotionType> ICP; extern cl::opt<unsigned> Verbosity; extern cl::opt<unsigned> ExecutionCountThreshold; static cl::opt<unsigned> ICPJTRemainingPercentThreshold( "icp-jt-remaining-percent-threshold", cl::desc("The percentage threshold against remaining unpromoted indirect " "call count for the promotion for jump tables"), cl::init(30), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<unsigned> ICPJTTotalPercentThreshold( "icp-jt-total-percent-threshold", cl::desc( "The percentage threshold against total count for the promotion for " "jump tables"), cl::init(5), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<unsigned> ICPCallsRemainingPercentThreshold( "icp-calls-remaining-percent-threshold", cl::desc("The percentage threshold against remaining unpromoted indirect " "call count for the promotion for calls"), cl::init(50), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<unsigned> ICPCallsTotalPercentThreshold( "icp-calls-total-percent-threshold", cl::desc( "The percentage threshold against total count for the promotion for " "calls"), cl::init(30), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<unsigned> ICPMispredictThreshold( "indirect-call-promotion-mispredict-threshold", cl::desc("misprediction threshold for skipping ICP on an " "indirect call"), cl::init(0), cl::cat(BoltOptCategory)); static cl::alias ICPMispredictThresholdAlias( "icp-mp-threshold", cl::desc("alias for --indirect-call-promotion-mispredict-threshold"), cl::aliasopt(ICPMispredictThreshold)); static cl::opt<bool> ICPUseMispredicts( "indirect-call-promotion-use-mispredicts", cl::desc("use misprediction frequency for determining whether or not ICP " "should be applied at a callsite. The " "-indirect-call-promotion-mispredict-threshold value will be used " "by this heuristic"), cl::cat(BoltOptCategory)); static cl::alias ICPUseMispredictsAlias( "icp-use-mp", cl::desc("alias for --indirect-call-promotion-use-mispredicts"), cl::aliasopt(ICPUseMispredicts)); static cl::opt<unsigned> ICPTopN("indirect-call-promotion-topn", cl::desc("limit number of targets to consider when doing indirect " "call promotion. 0 = no limit"), cl::init(3), cl::cat(BoltOptCategory)); static cl::alias ICPTopNAlias("icp-topn", cl::desc("alias for --indirect-call-promotion-topn"), cl::aliasopt(ICPTopN)); static cl::opt<unsigned> ICPCallsTopN( "indirect-call-promotion-calls-topn", cl::desc("limit number of targets to consider when doing indirect " "call promotion on calls. 0 = no limit"), cl::init(0), cl::cat(BoltOptCategory)); static cl::alias ICPCallsTopNAlias( "icp-calls-topn", cl::desc("alias for --indirect-call-promotion-calls-topn"), cl::aliasopt(ICPCallsTopN)); static cl::opt<unsigned> ICPJumpTablesTopN( "indirect-call-promotion-jump-tables-topn", cl::desc("limit number of targets to consider when doing indirect " "call promotion on jump tables. 0 = no limit"), cl::init(0), cl::cat(BoltOptCategory)); static cl::alias ICPJumpTablesTopNAlias( "icp-jt-topn", cl::desc("alias for --indirect-call-promotion-jump-tables-topn"), cl::aliasopt(ICPJumpTablesTopN)); static cl::opt<bool> EliminateLoads( "icp-eliminate-loads", cl::desc("enable load elimination using memory profiling data when " "performing ICP"), cl::init(true), cl::cat(BoltOptCategory)); static cl::opt<unsigned> ICPTopCallsites( "icp-top-callsites", cl::desc("optimize hottest calls until at least this percentage of all " "indirect calls frequency is covered. 0 = all callsites"), cl::init(99), cl::Hidden, cl::cat(BoltOptCategory)); static cl::list<std::string> ICPFuncsList("icp-funcs", cl::CommaSeparated, cl::desc("list of functions to enable ICP for"), cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<bool> ICPOldCodeSequence("icp-old-code-sequence", cl::desc("use old code sequence for promoted calls"), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<bool> ICPJumpTablesByTarget( "icp-jump-tables-targets", cl::desc( "for jump tables, optimize indirect jmp targets instead of indices"), cl::Hidden, cl::cat(BoltOptCategory)); static cl::alias ICPJumpTablesByTargetAlias("icp-jt-targets", cl::desc("alias for --icp-jump-tables-targets"), cl::aliasopt(ICPJumpTablesByTarget)); static cl::opt<bool> ICPPeelForInline( "icp-inline", cl::desc("only promote call targets eligible for inlining"), cl::Hidden, cl::cat(BoltOptCategory)); } // namespace opts #ifndef NDEBUG static bool verifyProfile(std::map<uint64_t, BinaryFunction> &BFs) { bool IsValid = true; for (auto &BFI : BFs) { BinaryFunction &BF = BFI.second; if (!BF.isSimple()) continue; for (const BinaryBasicBlock &BB : BF) { auto BI = BB.branch_info_begin(); for (BinaryBasicBlock *SuccBB : BB.successors()) { if (BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE && BI->Count > 0) { if (BB.getKnownExecutionCount() == 0 || SuccBB->getKnownExecutionCount() == 0) { BF.getBinaryContext().errs() << "BOLT-WARNING: profile verification failed after ICP for " "function " << BF << '\n'; IsValid = false; } } ++BI; } } } return IsValid; } #endif namespace llvm { namespace bolt { IndirectCallPromotion::Callsite::Callsite(BinaryFunction &BF, const IndirectCallProfile &ICP) : … { … } void IndirectCallPromotion::printDecision( llvm::raw_ostream &OS, std::vector<IndirectCallPromotion::Callsite> &Targets, unsigned N) const { … } // Get list of targets for a given call sorted by most frequently // called first. std::vector<IndirectCallPromotion::Callsite> IndirectCallPromotion::getCallTargets(BinaryBasicBlock &BB, const MCInst &Inst) const { … } IndirectCallPromotion::JumpTableInfoType IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB, MCInst &CallInst, MCInst *&TargetFetchInst, const JumpTable *JT) const { … } IndirectCallPromotion::SymTargetsType IndirectCallPromotion::findCallTargetSymbols(std::vector<Callsite> &Targets, size_t &N, BinaryBasicBlock &BB, MCInst &CallInst, MCInst *&TargetFetchInst) const { … } IndirectCallPromotion::MethodInfoType IndirectCallPromotion::maybeGetVtableSyms( BinaryBasicBlock &BB, MCInst &Inst, const SymTargetsType &SymTargets) const { … } std::vector<std::unique_ptr<BinaryBasicBlock>> IndirectCallPromotion::rewriteCall( BinaryBasicBlock &IndCallBlock, const MCInst &CallInst, MCPlusBuilder::BlocksVectorTy &&ICPcode, const std::vector<MCInst *> &MethodFetchInsns) const { … } BinaryBasicBlock * IndirectCallPromotion::fixCFG(BinaryBasicBlock &IndCallBlock, const bool IsTailCall, const bool IsJumpTable, IndirectCallPromotion::BasicBlocksVector &&NewBBs, const std::vector<Callsite> &Targets) const { … } size_t IndirectCallPromotion::canPromoteCallsite( const BinaryBasicBlock &BB, const MCInst &Inst, const std::vector<Callsite> &Targets, uint64_t NumCalls) { … } void IndirectCallPromotion::printCallsiteInfo( const BinaryBasicBlock &BB, const MCInst &Inst, const std::vector<Callsite> &Targets, const size_t N, uint64_t NumCalls) const { … } Error IndirectCallPromotion::runOnFunctions(BinaryContext &BC) { … } } // namespace bolt } // namespace llvm
Codebase Browser
llvm