#include "llvm/CodeGen/SelectionDAGISel.h"
#include "ScheduleDAGSDNodes.h"
#include "SelectionDAGBuilder.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/FastISel.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachinePassRegistry.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SchedulerRegistry.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/StackProtector.h"
#include "llvm/CodeGen/SwiftErrorValueTracking.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsWebAssembly.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PrintPasses.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Pass.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetIntrinsicInfo.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <limits>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>
usingnamespacellvm;
#define DEBUG_TYPE …
#define ISEL_DUMP_DEBUG_TYPE …
STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
STATISTIC(NumFastIselFailLowerArguments,
"Number of entry blocks where fast isel failed to lower arguments");
static cl::opt<int> EnableFastISelAbort(
"fast-isel-abort", cl::Hidden,
cl::desc("Enable abort calls when \"fast\" instruction selection "
"fails to lower an instruction: 0 disable the abort, 1 will "
"abort but for args, calls and terminators, 2 will also "
"abort for argument lowering, and 3 will never fallback "
"to SelectionDAG."));
static cl::opt<bool> EnableFastISelFallbackReport(
"fast-isel-report-on-fallback", cl::Hidden,
cl::desc("Emit a diagnostic when \"fast\" instruction selection "
"falls back to SelectionDAG."));
static cl::opt<bool>
UseMBPI("use-mbpi",
cl::desc("use Machine Branch Probability Info"),
cl::init(true), cl::Hidden);
#ifndef NDEBUG
static cl::opt<std::string>
FilterDAGBasicBlockName("filter-view-dags", cl::Hidden,
cl::desc("Only display the basic block whose name "
"matches this for all view-*-dags options"));
static cl::opt<bool>
ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before the first "
"dag combine pass"));
static cl::opt<bool>
ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before legalize types"));
static cl::opt<bool>
ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before the post "
"legalize types dag combine pass"));
static cl::opt<bool>
ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before legalize"));
static cl::opt<bool>
ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden,
cl::desc("Pop up a window to show dags before the second "
"dag combine pass"));
static cl::opt<bool>
ViewISelDAGs("view-isel-dags", cl::Hidden,
cl::desc("Pop up a window to show isel dags as they are selected"));
static cl::opt<bool>
ViewSchedDAGs("view-sched-dags", cl::Hidden,
cl::desc("Pop up a window to show sched dags as they are processed"));
static cl::opt<bool>
ViewSUnitDAGs("view-sunit-dags", cl::Hidden,
cl::desc("Pop up a window to show SUnit dags after they are processed"));
#else
static const bool ViewDAGCombine1 = …, ViewLegalizeTypesDAGs = …,
ViewDAGCombineLT = …, ViewLegalizeDAGs = …,
ViewDAGCombine2 = …, ViewISelDAGs = …,
ViewSchedDAGs = …, ViewSUnitDAGs = …;
#endif
#ifndef NDEBUG
#define ISEL_DUMP …
#else
#define ISEL_DUMP(X) …
#endif
MachinePassRegistry<RegisterScheduler::FunctionPassCtor>
RegisterScheduler::Registry;
static cl::opt<RegisterScheduler::FunctionPassCtor, false,
RegisterPassParser<RegisterScheduler>>
ISHeuristic("pre-RA-sched",
cl::init(&createDefaultScheduler), cl::Hidden,
cl::desc("Instruction schedulers available (before register"
" allocation):"));
static RegisterScheduler
defaultListDAGScheduler("default", "Best scheduler for the target",
createDefaultScheduler);
static bool dontUseFastISelFor(const Function &Fn) { … }
namespace llvm {
class OptLevelChanger { … };
ScheduleDAGSDNodes *createDefaultScheduler(SelectionDAGISel *IS,
CodeGenOptLevel OptLevel) { … }
}
MachineBasicBlock *
TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *MBB) const { … }
void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
SDNode *Node) const { … }
SelectionDAGISelLegacy::SelectionDAGISelLegacy(
char &ID, std::unique_ptr<SelectionDAGISel> S)
: … { … }
bool SelectionDAGISelLegacy::runOnMachineFunction(MachineFunction &MF) { … }
SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOptLevel OL)
: … { … }
SelectionDAGISel::~SelectionDAGISel() { … }
void SelectionDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const { … }
PreservedAnalyses
SelectionDAGISelPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) { … }
void SelectionDAGISel::initializeAnalysisResults(
MachineFunctionAnalysisManager &MFAM) { … }
void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { … }
bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { … }
static void reportFastISelFailure(MachineFunction &MF,
OptimizationRemarkEmitter &ORE,
OptimizationRemarkMissed &R,
bool ShouldAbort) { … }
static void preserveFakeUses(BasicBlock::iterator Begin,
BasicBlock::iterator End) { … }
void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
BasicBlock::const_iterator End,
bool &HadTailCall) { … }
void SelectionDAGISel::ComputeLiveOutVRegInfo() { … }
void SelectionDAGISel::CodeGenAndEmitDAG() { … }
namespace {
class ISelUpdater : public SelectionDAG::DAGUpdateListener { … };
}
void SelectionDAGISel::EnforceNodeIdInvariant(SDNode *Node) { … }
void SelectionDAGISel::InvalidateNodeId(SDNode *N) { … }
int SelectionDAGISel::getUninvalidatedNodeId(SDNode *N) { … }
void SelectionDAGISel::DoInstructionSelection() { … }
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI) { … }
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB,
const CatchPadInst *CPI) { … }
bool SelectionDAGISel::PrepareEHLandingPad() { … }
void SelectionDAGISel::reportIPToStateForBlocks(MachineFunction *MF) { … }
static bool isFoldedOrDeadInstruction(const Instruction *I,
const FunctionLoweringInfo &FuncInfo) { … }
static bool processIfEntryValueDbgDeclare(FunctionLoweringInfo &FuncInfo,
const Value *Arg, DIExpression *Expr,
DILocalVariable *Var,
DebugLoc DbgLoc) { … }
static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo,
const Value *Address, DIExpression *Expr,
DILocalVariable *Var, DebugLoc DbgLoc) { … }
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo) { … }
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo,
FunctionVarLocs const *FnVarLocs) { … }
void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { … }
void
SelectionDAGISel::FinishBasicBlock() { … }
ScheduleDAGSDNodes *SelectionDAGISel::CreateScheduler() { … }
bool SelectionDAGISel::CheckAndMask(SDValue LHS, ConstantSDNode *RHS,
int64_t DesiredMaskS) const { … }
bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
int64_t DesiredMaskS) const { … }
void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops,
const SDLoc &DL) { … }
static SDNode *findGlueUse(SDNode *N) { … }
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
bool IgnoreChains) { … }
bool SelectionDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
SDNode *Root) const { … }
bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
CodeGenOptLevel OptLevel,
bool IgnoreChains) { … }
void SelectionDAGISel::Select_INLINEASM(SDNode *N) { … }
void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) { … }
void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) { … }
void SelectionDAGISel::Select_UNDEF(SDNode *N) { … }
void SelectionDAGISel::Select_FAKE_USE(SDNode *N) { … }
void SelectionDAGISel::Select_FREEZE(SDNode *N) { … }
void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) { … }
void SelectionDAGISel::Select_MEMBARRIER(SDNode *N) { … }
void SelectionDAGISel::Select_CONVERGENCECTRL_ANCHOR(SDNode *N) { … }
void SelectionDAGISel::Select_CONVERGENCECTRL_ENTRY(SDNode *N) { … }
void SelectionDAGISel::Select_CONVERGENCECTRL_LOOP(SDNode *N) { … }
void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops,
SDValue OpVal, SDLoc DL) { … }
void SelectionDAGISel::Select_STACKMAP(SDNode *N) { … }
void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t
GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static MVT::SimpleValueType
getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex) { … }
void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(SDNode *N) { … }
void SelectionDAGISel::UpdateChains(
SDNode *NodeToMatch, SDValue InputChain,
SmallVectorImpl<SDNode *> &ChainNodesMatched, bool isMorphNodeTo) { … }
static SDValue
HandleMergeInputChains(SmallVectorImpl<SDNode*> &ChainNodesMatched,
SelectionDAG *CurDAG) { … }
SDNode *SelectionDAGISel::
MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
ArrayRef<SDValue> Ops, unsigned EmitNodeInfo) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckChildSame(
const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
unsigned ChildNo) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckPatternPredicate(unsigned Opcode, const unsigned char *MatcherTable,
unsigned &MatcherIndex, const SelectionDAGISel &SDISel) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckNodePredicate(unsigned Opcode, const unsigned char *MatcherTable,
unsigned &MatcherIndex, const SelectionDAGISel &SDISel,
SDNode *N) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDNode *N) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckType(MVT::SimpleValueType VT,
SDValue N,
const TargetLowering *TLI,
const DataLayout &DL) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckChildType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI,
const DataLayout &DL, unsigned ChildNo) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDValue N) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDValue N) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDValue N, const TargetLowering *TLI, const DataLayout &DL) { … }
static uint64_t decodeSignRotatedValue(uint64_t V) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDValue N) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDValue N, unsigned ChildNo) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
SDValue N, const SelectionDAGISel &SDISel) { … }
LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
const SelectionDAGISel &SDISel) { … }
static unsigned IsPredicateKnownToFail(const unsigned char *Table,
unsigned Index, SDValue N,
bool &Result,
const SelectionDAGISel &SDISel,
SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) { … }
namespace {
struct MatchScope { … };
class MatchStateUpdater : public SelectionDAG::DAGUpdateListener
{ … };
}
void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
const unsigned char *MatcherTable,
unsigned TableSize) { … }
bool SelectionDAGISel::mayRaiseFPException(SDNode *N) const { … }
bool SelectionDAGISel::isOrEquivalentToAdd(const SDNode *N) const { … }
void SelectionDAGISel::CannotYetSelect(SDNode *N) { … }