//===- bolt/Passes/BinaryPasses.cpp - Binary-level passes -----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements multiple passes for binary optimization and analysis. // //===----------------------------------------------------------------------===// #include "bolt/Passes/BinaryPasses.h" #include "bolt/Core/FunctionLayout.h" #include "bolt/Core/ParallelUtilities.h" #include "bolt/Passes/ReorderAlgorithm.h" #include "bolt/Passes/ReorderFunctions.h" #include "llvm/Support/CommandLine.h" #include <atomic> #include <mutex> #include <numeric> #include <vector> #define DEBUG_TYPE … usingnamespacellvm; usingnamespacebolt; static const char *dynoStatsOptName(const bolt::DynoStats::Category C) { … } namespace opts { extern cl::OptionCategory BoltCategory; extern cl::OptionCategory BoltOptCategory; extern cl::opt<unsigned> Verbosity; extern cl::opt<bool> EnableBAT; extern cl::opt<unsigned> ExecutionCountThreshold; extern cl::opt<bool> UpdateDebugSections; extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions; enum DynoStatsSortOrder : char { … }; static cl::opt<DynoStatsSortOrder> DynoStatsSortOrderOpt( "print-sorted-by-order", cl::desc("use ascending or descending order when printing functions " "ordered by dyno stats"), cl::init(DynoStatsSortOrder::Descending), cl::cat(BoltOptCategory)); cl::list<std::string> HotTextMoveSections("hot-text-move-sections", cl::desc("list of sections containing functions used for hugifying hot text. " "BOLT makes sure these functions are not placed on the same page as " "the hot text. (default=\'.stub,.mover\')."), cl::value_desc("sec1,sec2,sec3,..."), cl::CommaSeparated, cl::ZeroOrMore, cl::cat(BoltCategory)); bool isHotTextMover(const BinaryFunction &Function) { … } static cl::opt<bool> MinBranchClusters( "min-branch-clusters", cl::desc("use a modified clustering algorithm geared towards minimizing " "branches"), cl::Hidden, cl::cat(BoltOptCategory)); static cl::list<Peepholes::PeepholeOpts> Peepholes( "peepholes", cl::CommaSeparated, cl::desc("enable peephole optimizations"), cl::value_desc("opt1,opt2,opt3,..."), cl::values(clEnumValN(Peepholes::PEEP_NONE, "none", "disable peepholes"), clEnumValN(Peepholes::PEEP_DOUBLE_JUMPS, "double-jumps", "remove double jumps when able"), clEnumValN(Peepholes::PEEP_TAILCALL_TRAPS, "tailcall-traps", "insert tail call traps"), clEnumValN(Peepholes::PEEP_USELESS_BRANCHES, "useless-branches", "remove useless conditional branches"), clEnumValN(Peepholes::PEEP_ALL, "all", "enable all peephole optimizations")), cl::ZeroOrMore, cl::cat(BoltOptCategory)); static cl::opt<unsigned> PrintFuncStat("print-function-statistics", cl::desc("print statistics about basic block ordering"), cl::init(0), cl::cat(BoltOptCategory)); static cl::opt<bool> PrintLargeFunctions( "print-large-functions", cl::desc("print functions that could not be overwritten due to excessive " "size"), cl::init(false), cl::cat(BoltOptCategory)); static cl::list<bolt::DynoStats::Category> PrintSortedBy("print-sorted-by", cl::CommaSeparated, cl::desc("print functions sorted by order of dyno stats"), cl::value_desc("key1,key2,key3,..."), cl::values( #define D(name, description, ...) \ clEnumValN(bolt::DynoStats::name, dynoStatsOptName(bolt::DynoStats::name), \ description), REAL_DYNO_STATS #undef D clEnumValN(bolt::DynoStats::LAST_DYNO_STAT, "all", "sorted by all names")), cl::ZeroOrMore, cl::cat(BoltOptCategory)); static cl::opt<bool> PrintUnknown("print-unknown", cl::desc("print names of functions with unknown control flow"), cl::cat(BoltCategory), cl::Hidden); static cl::opt<bool> PrintUnknownCFG("print-unknown-cfg", cl::desc("dump CFG of functions with unknown control flow"), cl::cat(BoltCategory), cl::ReallyHidden); // Please MSVC19 with a forward declaration: otherwise it reports an error about // an undeclared variable inside a callback. extern cl::opt<bolt::ReorderBasicBlocks::LayoutType> ReorderBlocks; cl::opt<bolt::ReorderBasicBlocks::LayoutType> ReorderBlocks( "reorder-blocks", cl::desc("change layout of basic blocks in a function"), cl::init(bolt::ReorderBasicBlocks::LT_NONE), cl::values( clEnumValN(bolt::ReorderBasicBlocks::LT_NONE, "none", "do not reorder basic blocks"), clEnumValN(bolt::ReorderBasicBlocks::LT_REVERSE, "reverse", "layout blocks in reverse order"), clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE, "normal", "perform optimal layout based on profile"), clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_BRANCH, "branch-predictor", "perform optimal layout prioritizing branch " "predictions"), clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_CACHE, "cache", "perform optimal layout prioritizing I-cache " "behavior"), clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_CACHE_PLUS, "cache+", "perform layout optimizing I-cache behavior"), clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_EXT_TSP, "ext-tsp", "perform layout optimizing I-cache behavior"), clEnumValN(bolt::ReorderBasicBlocks::LT_OPTIMIZE_SHUFFLE, "cluster-shuffle", "perform random layout of clusters")), cl::ZeroOrMore, cl::cat(BoltOptCategory), cl::callback([](const bolt::ReorderBasicBlocks::LayoutType &option) { … })); static cl::opt<unsigned> ReportBadLayout( "report-bad-layout", cl::desc("print top <uint> functions with suboptimal code layout on input"), cl::init(0), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<bool> ReportStaleFuncs("report-stale", cl::desc("print the list of functions with stale profile"), cl::Hidden, cl::cat(BoltOptCategory)); enum SctcModes : char { … }; static cl::opt<SctcModes> SctcMode("sctc-mode", cl::desc("mode for simplify conditional tail calls"), cl::init(SctcAlways), cl::values(clEnumValN(SctcAlways, "always", "always perform sctc"), clEnumValN(SctcPreserveDirection, "preserve", "only perform sctc when branch direction is " "preserved"), clEnumValN(SctcHeuristic, "heuristic", "use branch prediction data to control sctc")), cl::ZeroOrMore, cl::cat(BoltOptCategory)); static cl::opt<unsigned> StaleThreshold("stale-threshold", cl::desc( "maximum percentage of stale functions to tolerate (default: 100)"), cl::init(100), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<unsigned> TSPThreshold( "tsp-threshold", cl::desc( "maximum number of hot basic blocks in a function for which to use " "a precise TSP solution while re-ordering basic blocks"), cl::init(10), cl::Hidden, cl::cat(BoltOptCategory)); static cl::opt<unsigned> TopCalledLimit( "top-called-limit", cl::desc("maximum number of functions to print in top called " "functions section"), cl::init(100), cl::Hidden, cl::cat(BoltCategory)); } // namespace opts namespace llvm { namespace bolt { bool BinaryFunctionPass::shouldOptimize(const BinaryFunction &BF) const { … } bool BinaryFunctionPass::shouldPrint(const BinaryFunction &BF) const { … } void NormalizeCFG::runOnFunction(BinaryFunction &BF) { … } Error NormalizeCFG::runOnFunctions(BinaryContext &BC) { … } void EliminateUnreachableBlocks::runOnFunction(BinaryFunction &Function) { … } Error EliminateUnreachableBlocks::runOnFunctions(BinaryContext &BC) { … } bool ReorderBasicBlocks::shouldPrint(const BinaryFunction &BF) const { … } bool ReorderBasicBlocks::shouldOptimize(const BinaryFunction &BF) const { … } Error ReorderBasicBlocks::runOnFunctions(BinaryContext &BC) { … } bool ReorderBasicBlocks::modifyFunctionLayout(BinaryFunction &BF, LayoutType Type, bool MinBranchClusters) const { … } Error FixupBranches::runOnFunctions(BinaryContext &BC) { … } Error FinalizeFunctions::runOnFunctions(BinaryContext &BC) { … } Error CheckLargeFunctions::runOnFunctions(BinaryContext &BC) { … } bool CheckLargeFunctions::shouldOptimize(const BinaryFunction &BF) const { … } Error LowerAnnotations::runOnFunctions(BinaryContext &BC) { … } // Check for dirty state in MCSymbol objects that might be a consequence // of running calculateEmittedSize() in parallel, during split functions // pass. If an inconsistent state is found (symbol already registered or // already defined), clean it. Error CleanMCState::runOnFunctions(BinaryContext &BC) { … } // This peephole fixes jump instructions that jump to another basic // block with a single jump instruction, e.g. // // B0: ... // jmp B1 (or jcc B1) // // B1: jmp B2 // // -> // // B0: ... // jmp B2 (or jcc B2) // static uint64_t fixDoubleJumps(BinaryFunction &Function, bool MarkInvalid) { … } bool SimplifyConditionalTailCalls::shouldRewriteBranch( const BinaryBasicBlock *PredBB, const MCInst &CondBranch, const BinaryBasicBlock *BB, const bool DirectionFlag) { … } uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) { … } Error SimplifyConditionalTailCalls::runOnFunctions(BinaryContext &BC) { … } uint64_t ShortenInstructions::shortenInstructions(BinaryFunction &Function) { … } Error ShortenInstructions::runOnFunctions(BinaryContext &BC) { … } void Peepholes::addTailcallTraps(BinaryFunction &Function) { … } void Peepholes::removeUselessCondBranches(BinaryFunction &Function) { … } Error Peepholes::runOnFunctions(BinaryContext &BC) { … } bool SimplifyRODataLoads::simplifyRODataLoads(BinaryFunction &BF) { … } Error SimplifyRODataLoads::runOnFunctions(BinaryContext &BC) { … } Error AssignSections::runOnFunctions(BinaryContext &BC) { … } Error PrintProfileStats::runOnFunctions(BinaryContext &BC) { … } Error PrintProgramStats::runOnFunctions(BinaryContext &BC) { … } Error InstructionLowering::runOnFunctions(BinaryContext &BC) { … } Error StripRepRet::runOnFunctions(BinaryContext &BC) { … } Error InlineMemcpy::runOnFunctions(BinaryContext &BC) { … } bool SpecializeMemcpy1::shouldOptimize(const BinaryFunction &Function) const { … } std::set<size_t> SpecializeMemcpy1::getCallSitesToOptimize( const BinaryFunction &Function) const { … } Error SpecializeMemcpy1::runOnFunctions(BinaryContext &BC) { … } void RemoveNops::runOnFunction(BinaryFunction &BF) { … } Error RemoveNops::runOnFunctions(BinaryContext &BC) { … } } // namespace bolt } // namespace llvm
Codebase Browser
llvm