//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // /// \file /// SI Implementation of TargetInstrInfo. // //===----------------------------------------------------------------------===// #include "SIInstrInfo.h" #include "AMDGPU.h" #include "AMDGPUInstrInfo.h" #include "GCNHazardRecognizer.h" #include "GCNSubtarget.h" #include "SIMachineFunctionInfo.h" #include "Utils/AMDGPUBaseInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineScheduler.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/MC/MCContext.h" #include "llvm/Support/CommandLine.h" #include "llvm/Target/TargetMachine.h" usingnamespacellvm; #define DEBUG_TYPE … #define GET_INSTRINFO_CTOR_DTOR #include "AMDGPUGenInstrInfo.inc" namespace llvm::AMDGPU { #define GET_D16ImageDimIntrinsics_IMPL #define GET_ImageDimIntrinsicTable_IMPL #define GET_RsrcIntrinsics_IMPL #include "AMDGPUGenSearchableTables.inc" } // namespace llvm::AMDGPU // Must be at least 4 to be able to branch over minimum unconditional branch // code. This is only for making it possible to write reasonably small tests for // long branches. static cl::opt<unsigned> BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), cl::desc("Restrict range of branch instructions (DEBUG)")); static cl::opt<bool> Fix16BitCopies( "amdgpu-fix-16-bit-physreg-copies", cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), cl::init(true), cl::ReallyHidden); SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) : … { … } //===----------------------------------------------------------------------===// // TargetInstrInfo callbacks //===----------------------------------------------------------------------===// static unsigned getNumOperandsNoGlue(SDNode *Node) { … } /// Returns true if both nodes have the same value for the given /// operand \p Op, or if both nodes do not have this operand. static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { … } static bool canRemat(const MachineInstr &MI) { … } bool SIInstrInfo::isReallyTriviallyReMaterializable( const MachineInstr &MI) const { … } // Returns true if the scalar result of a VALU instruction depends on exec. static bool resultDependsOnExec(const MachineInstr &MI) { … } bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const { … } bool SIInstrInfo::isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo, MachineCycleInfo *CI) const { … } bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0, int64_t &Offset1) const { … } static bool isStride64(unsigned Opc) { … } bool SIInstrInfo::getMemOperandsWithOffsetWidth( const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const { … } static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef<const MachineOperand *> BaseOps1, const MachineInstr &MI2, ArrayRef<const MachineOperand *> BaseOps2) { … } bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const { … } // FIXME: This behaves strangely. If, for example, you have 32 load + stores, // the first 16 loads will be interleaved with the stores, and the next 16 will // be clustered as expected. It should really split into 2 16 store batches. // // Loads are clustered until this returns false, rather than trying to schedule // groups of stores. This also means we have to deal with saying different // address space loads should be clustered, and ones which might cause bank // conflicts. // // This might be deprecated so it might not be worth that much effort to fix. bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0, int64_t Offset1, unsigned NumLoads) const { … } static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, const char *Msg = "illegal VGPR to SGPR copy") { … } /// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not /// possible to have a direct copy in these cases on GFX908, so an intermediate /// VGPR copy is required. static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, RegScavenger &RS, bool RegsOverlap, Register ImpDefSuperReg = Register(), Register ImpUseSuperReg = Register()) { … } static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RC, bool Forward) { … } void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc, bool RenamableDest, bool RenamableSrc) const { … } int SIInstrInfo::commuteOpcode(unsigned Opcode) const { … } void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL, Register DestReg, int64_t Value) const { … } const TargetRegisterClass * SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { … } void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DstReg, ArrayRef<MachineOperand> Cond, Register TrueReg, Register FalseReg) const { … } Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SrcReg, int Value) const { … } Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register SrcReg, int Value) const { … } unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { … } const MCInstrDesc & SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, bool IsIndirectSrc) const { … } static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { … } static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { … } static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { … } const MCInstrDesc & SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, bool IsSGPR) const { … } static unsigned getSGPRSpillSaveOpcode(unsigned Size) { … } static unsigned getVGPRSpillSaveOpcode(unsigned Size) { … } static unsigned getAGPRSpillSaveOpcode(unsigned Size) { … } static unsigned getAVSpillSaveOpcode(unsigned Size) { … } static unsigned getWWMRegSpillSaveOpcode(unsigned Size, bool IsVectorSuperClass) { … } static unsigned getVectorRegSpillSaveOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &MFI) { … } void SIInstrInfo::storeRegToStackSlot( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const { … } static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { … } static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { … } static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { … } static unsigned getAVSpillRestoreOpcode(unsigned Size) { … } static unsigned getWWMRegSpillRestoreOpcode(unsigned Size, bool IsVectorSuperClass) { … } static unsigned getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &MFI) { … } void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const { … } void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const { … } void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned Quantity) const { … } void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { … } MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI, MachineBasicBlock &MBB, MachineInstr &MI, const DebugLoc &DL) const { … } unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { … } bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { … } void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, Register DestReg, unsigned SubIdx, const MachineInstr &Orig, const TargetRegisterInfo &RI) const { … } std::pair<MachineInstr*, MachineInstr*> SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { … } std::optional<DestSourcePair> SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { … } bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0, unsigned Src0OpName, MachineOperand &Src1, unsigned Src1OpName) const { … } static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, MachineOperand &RegOp, MachineOperand &NonRegOp) { … } MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned Src0Idx, unsigned Src1Idx) const { … } // This needs to be implemented because the source modifiers may be inserted // between the true commutable operands, and the base // TargetInstrInfo::commuteInstruction uses it. bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0, unsigned &SrcOpIdx1) const { … } bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0, unsigned &SrcOpIdx1) const { … } bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, int64_t BrOffset) const { … } MachineBasicBlock * SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const { … } bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const { … } void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &DestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const { … } unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { … } SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { … } bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl<MachineOperand> &Cond, bool AllowModify) const { … } bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl<MachineOperand> &Cond, bool AllowModify) const { … } unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, int *BytesRemoved) const { … } // Copy the flags onto the implicit condition register operand. static void preserveCondRegFlags(MachineOperand &CondReg, const MachineOperand &OrigCond) { … } unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const { … } bool SIInstrInfo::reverseBranchCondition( SmallVectorImpl<MachineOperand> &Cond) const { … } bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, ArrayRef<MachineOperand> Cond, Register DstReg, Register TrueReg, Register FalseReg, int &CondCycles, int &TrueCycles, int &FalseCycles) const { … } void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DstReg, ArrayRef<MachineOperand> Cond, Register TrueReg, Register FalseReg) const { … } bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) { … } static constexpr unsigned ModifierOpNames[] = …; void SIInstrInfo::removeModOperands(MachineInstr &MI) const { … } bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const { … } static bool memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, ArrayRef<const MachineOperand *> BaseOps2) { … } static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA, LocationSize WidthB, int OffsetB) { … } bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, const MachineInstr &MIb) const { … } bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const { … } static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI, int64_t &Imm, MachineInstr **DefMI = nullptr) { … } static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm, MachineInstr **DefMI = nullptr) { … } static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, MachineInstr &NewMI) { … } MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const { … } // It's not generally safe to move VALU instructions across these since it will // start using the register as a base index rather than directly. // XXX - Why isn't hasSideEffects sufficient for these? static bool changesVGPRIndexingMode(const MachineInstr &MI) { … } bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const { … } bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { … } bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { … } bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { … } bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const { … } bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { … } bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const { … } bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const { … } static bool compareMachineOp(const MachineOperand &Op0, const MachineOperand &Op1) { … } bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, const MachineOperand &MO) const { … } bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { … } bool SIInstrInfo::hasModifiers(unsigned Opcode) const { … } bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, unsigned OpName) const { … } bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { … } bool SIInstrInfo::canShrink(const MachineInstr &MI, const MachineRegisterInfo &MRI) const { … } // Set VCC operand with all flags from \p Orig, except for setting it as // implicit. static void copyFlagsToImplicitVCC(MachineInstr &MI, const MachineOperand &Orig) { … } MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, unsigned Op32) const { … } bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, const MachineOperand &MO, const MCOperandInfo &OpInfo) const { … } static Register findImplicitSGPRRead(const MachineInstr &MI) { … } static bool shouldReadExec(const MachineInstr &MI) { … } static bool isRegOrFI(const MachineOperand &MO) { … } static bool isSubRegOf(const SIRegisterInfo &TRI, const MachineOperand &SuperVec, const MachineOperand &SubReg) { … } // Verify the illegal copy from vector register to SGPR for generic opcode COPY bool SIInstrInfo::verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI, StringRef &ErrInfo) const { … } bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const { … } // It is more readable to list mapped opcodes on the same line. // clang-format off unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { … } // clang-format on void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register Reg, bool IsSCCLive, SlotIndexes *Indexes) const { … } void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register Reg, SlotIndexes *Indexes) const { … } static const TargetRegisterClass * adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI, const MachineRegisterInfo &MRI, const MCInstrDesc &TID, unsigned RCID, bool IsAllocatable) { … } const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const { … } const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, unsigned OpNo) const { … } void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { … } unsigned SIInstrInfo::buildExtractSubReg( MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, unsigned SubIdx, const TargetRegisterClass *SubRC) const { … } MachineOperand SIInstrInfo::buildExtractSubRegOrImm( MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI, const MachineOperand &Op, const TargetRegisterClass *SuperRC, unsigned SubIdx, const TargetRegisterClass *SubRC) const { … } // Change the order of operands from (0, 1, 2) to (0, 2, 1) void SIInstrInfo::swapOperands(MachineInstr &Inst) const { … } bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, const MCOperandInfo &OpInfo, const MachineOperand &MO) const { … } bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, const MCOperandInfo &OpInfo, const MachineOperand &MO) const { … } bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, const MachineOperand *MO) const { … } void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const { … } // Legalize VOP3 operands. All operand types are supported for any operand // but only one literal constant and only starting from GFX10. void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const { … } Register SIInstrInfo::readlaneVGPRToSGPR( Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI, const TargetRegisterClass *DstRC /*=nullptr*/) const { … } void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const { … } bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const { … } // FIXME: Remove this when SelectionDAG is obsoleted. void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const { … } void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, MachineBasicBlock::iterator I, const TargetRegisterClass *DstRC, MachineOperand &Op, MachineRegisterInfo &MRI, const DebugLoc &DL) const { … } // Emit the actual waterfall loop, executing the wrapped instruction for each // unique value of \p ScalarOps across all lanes. In the best case we execute 1 // iteration, in the worst case we execute 64 (once per lane). static void emitLoadScalarOpsFromVGPRLoop( const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL, ArrayRef<MachineOperand *> ScalarOps) { … } // Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register // with SGPRs by iterating over all unique values across all lanes. // Returns the loop basic block that now contains \p MI. static MachineBasicBlock * loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, ArrayRef<MachineOperand *> ScalarOps, MachineDominatorTree *MDT, MachineBasicBlock::iterator Begin = nullptr, MachineBasicBlock::iterator End = nullptr) { … } // Extract pointer from Rsrc and return a zero-value Rsrc replacement. static std::tuple<unsigned, unsigned> extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { … } MachineBasicBlock * SIInstrInfo::legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT) const { … } void SIInstrWorklist::insert(MachineInstr *MI) { … } bool SIInstrWorklist::isDeferred(MachineInstr *MI) { … } void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const { … } void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst) const { … } // Add/sub require special handling to deal with carry outs. std::pair<bool, MachineBasicBlock *> SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst, MachineDominatorTree *MDT) const { … } void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst, MachineDominatorTree *MDT) const { … } void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const { … } void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const { … } void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst, unsigned Opcode) const { … } void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst, unsigned Opcode) const { … } void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst, unsigned Opcode, bool Swap) const { … } // There is not a vector equivalent of s_mul_u64. For this reason, we need to // split the s_mul_u64 in 32-bit vector multiplications. void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst, MachineDominatorTree *MDT) const { … } // Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector // multiplications. void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst, MachineDominatorTree *MDT) const { … } void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst, unsigned Opcode, MachineDominatorTree *MDT) const { … } void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst, MachineDominatorTree *MDT) const { … } void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist, MachineInstr &Inst) const { … } void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const { … } void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst, unsigned Opcode, MachineDominatorTree *MDT) const { … } void SIInstrInfo::addUsersToMoveToVALUWorklist( Register DstReg, MachineRegisterInfo &MRI, SIInstrWorklist &Worklist) const { … } void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI, MachineInstr &Inst) const { … } void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, MachineInstr &SCCDefInst, SIInstrWorklist &Worklist, Register NewCond) const { … } // Instructions that use SCC may be converted to VALU instructions. When that // happens, the SCC register is changed to VCC_LO. The instruction that defines // SCC must be changed to an instruction that defines VCC. This function makes // sure that the instruction that defines SCC is added to the moveToVALU // worklist. void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst, SIInstrWorklist &Worklist) const { … } const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( const MachineInstr &Inst) const { … } // Find the one SGPR operand we are allowed to use. Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const { … } MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, unsigned OperandName) const { … } uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { … } uint64_t SIInstrInfo::getScratchRsrcWords23() const { … } bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { … } bool SIInstrInfo::isHighLatencyDef(int Opc) const { … } unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, int &FrameIndex) const { … } unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const { … } Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const { … } Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const { … } unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { … } unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { … } bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { … } ArrayRef<std::pair<int, const char *>> SIInstrInfo::getSerializableTargetIndices() const { … } /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The /// post-RA version of misched uses CreateTargetMIHazardRecognizer. ScheduleHazardRecognizer * SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const { … } /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer /// pass. ScheduleHazardRecognizer * SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { … } // Called during: // - pre-RA scheduling and post-RA scheduling ScheduleHazardRecognizer * SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II, const ScheduleDAGMI *DAG) const { … } std::pair<unsigned, unsigned> SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { … } ArrayRef<std::pair<unsigned, const char *>> SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { … } ArrayRef<std::pair<MachineMemOperand::Flags, const char *>> SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const { … } unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg, const MachineFunction &MF) const { … } bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI, Register Reg) const { … } MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg) const { … } MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DestReg, RegScavenger &RS) const { … } bool SIInstrInfo::isKillTerminator(unsigned Opcode) { … } const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { … } bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const { … } unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) { … } void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { … } bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { … } // Given Imm, split it into the values to put into the SOffset and ImmOffset // fields in an MUBUF instruction. Return false if it is not possible (due to a // hardware bug needing a workaround). // // The required alignment ensures that individual address components remain // aligned if they are aligned to begin with. It also ensures that additional // offsets within the given alignment can be added to the resulting ImmOffset. bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, Align Alignment) const { … } // Depending on the used address space and instructions, some immediate offsets // are allowed and some are not. // Pre-GFX12, flat instruction offsets can only be non-negative, global and // scratch instruction offsets can also be negative. On GFX12, offsets can be // negative for all variants. // // There are several bugs related to these offsets: // On gfx10.1, flat instructions that go into the global address space cannot // use an offset. // // For scratch instructions, the address can be either an SGPR or a VGPR. // The following offsets can be used, depending on the architecture (x means // cannot be used): // +----------------------------+------+------+ // | Address-Mode | SGPR | VGPR | // +----------------------------+------+------+ // | gfx9 | | | // | negative, 4-aligned offset | x | ok | // | negative, unaligned offset | x | ok | // +----------------------------+------+------+ // | gfx10 | | | // | negative, 4-aligned offset | ok | ok | // | negative, unaligned offset | ok | x | // +----------------------------+------+------+ // | gfx10.3 | | | // | negative, 4-aligned offset | ok | ok | // | negative, unaligned offset | ok | ok | // +----------------------------+------+------+ // // This function ignores the addressing mode, so if an offset cannot be used in // one addressing mode, it is considered illegal. bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, uint64_t FlatVariant) const { … } // See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not. std::pair<int64_t, int64_t> SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, uint64_t FlatVariant) const { … } bool SIInstrInfo::allowNegativeFlatOffset(uint64_t FlatVariant) const { … } static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) { … } bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { … } #define GENERATE_RENAMED_GFX9_CASES(OPCODE) … static bool isRenamedInGFX9(int Opcode) { … } int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { … } static TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { … } TargetInstrInfo::RegSubRegPair llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { … } // Try to find the definition of reg:subreg in subreg-manipulation pseudos // Following a subreg of reg:subreg isn't supported static bool followSubRegDef(MachineInstr &MI, TargetInstrInfo::RegSubRegPair &RSR) { … } MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, MachineRegisterInfo &MRI) { … } bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI, const MachineInstr &UseMI) { … } bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, Register VReg, const MachineInstr &DefMI) { … } MachineInstr *SIInstrInfo::createPHIDestinationCopy( MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, const DebugLoc &DL, Register Src, Register Dst) const { … } MachineInstr *SIInstrInfo::createPHISourceCopy( MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { … } bool llvm::SIInstrInfo::isWave32() const { … } MachineInstr *SIInstrInfo::foldMemoryOperandImpl( MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, VirtRegMap *VRM) const { … } unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, const MachineInstr &MI, unsigned *PredCost) const { … } InstructionUniformity SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const { … } InstructionUniformity SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const { … } unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { … } bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, Register &SrcReg2, int64_t &CmpMask, int64_t &CmpValue) const { … } bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask, int64_t CmpValue, const MachineRegisterInfo *MRI) const { … } void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI, unsigned OpName) const { … }
Codebase Browser
llvm