#include "AMDGPUInstructionSelector.h"
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPURegisterBankInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include <optional>
#define DEBUG_TYPE …
usingnamespacellvm;
usingnamespaceMIPatternMatch;
#define GET_GLOBALISEL_IMPL
#define AMDGPUSubtarget …
#include "AMDGPUGenGlobalISel.inc"
#undef GET_GLOBALISEL_IMPL
#undef AMDGPUSubtarget
AMDGPUInstructionSelector::AMDGPUInstructionSelector(
const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI,
const AMDGPUTargetMachine &TM)
: … { … }
const char *AMDGPUInstructionSelector::getName() { … }
void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
CodeGenCoverage *CoverageInfo,
ProfileSummaryInfo *PSI,
BlockFrequencyInfo *BFI) { … }
static Register getWaveAddress(const MachineInstr *Def) { … }
bool AMDGPUInstructionSelector::isVCC(Register Reg,
const MachineRegisterInfo &MRI) const { … }
bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
unsigned NewOpc) const { … }
bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const { … }
MachineOperand
AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
const TargetRegisterClass &SubRC,
unsigned SubIdx) const { … }
static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) { … }
bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const { … }
static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
const GCNSubtarget &ST) { … }
int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
unsigned Size) const { … }
bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectIntrinsicCmp(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
MachineInstr &MI, Intrinsic::ID IntrID) const { … }
static unsigned gwsIntrinToOpcode(unsigned IntrID) { … }
bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
Intrinsic::ID IID) const { … }
bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
bool IsAppend) const { … }
bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectSBarrier(MachineInstr &MI) const { … }
static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
bool &IsTexFail) { … }
bool AMDGPUInstructionSelector::selectImageIntrinsic(
MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const { … }
bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const { … }
static bool shouldUseAndMask(unsigned Size, unsigned &Mask) { … }
const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
Register Reg, const MachineRegisterInfo &MRI,
const TargetRegisterInfo &TRI) const { … }
bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const { … }
static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) { … }
static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) { … }
static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
Register &Out) { … }
bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const { … }
static bool isConstant(const MachineInstr &MI) { … }
void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const { … }
bool AMDGPUInstructionSelector::isSGPR(Register Reg) const { … }
bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const { … }
void AMDGPUInstructionSelector::initM0(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
MachineInstr &I) const { … }
static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) { … }
bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
MachineInstr &I) const { … }
bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const { … }
static std::pair<Register, unsigned>
computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
const TargetRegisterClass *SuperRC, Register IdxReg,
unsigned EltSize, GISelKnownBits &KnownBits) { … }
bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const { … }
static Register matchZeroExtendFromS32(MachineRegisterInfo &MRI, Register Reg) { … }
bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{ … }
bool AMDGPUInstructionSelector::selectBVHIntrinsic(MachineInstr &MI) const{ … }
bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const { … }
bool AMDGPUInstructionSelector::select(MachineInstr &I) { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const { … }
std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const { … }
Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
bool ForceVGPR) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const { … }
std::pair<Register, unsigned>
AMDGPUInstructionSelector::selectVOP3PModsImpl(
Register Src, const MachineRegisterInfo &MRI, bool IsDOT) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3PModsNeg(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
MachineOperand &Root) const { … }
static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
MachineInstr *InsertPt,
MachineRegisterInfo &MRI) { … }
static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
SmallVectorImpl<Register> &Elts, Register &Src,
MachineInstr *InsertPt,
MachineRegisterInfo &MRI) { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const { … }
bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
Register &Base,
Register *SOffset,
int64_t *Offset) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const { … }
std::pair<Register, int>
AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root,
uint64_t FlatVariant) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const { … }
bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
Register VAddr, Register SAddr, uint64_t ImmOffset) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const { … }
bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
int64_t Offset) const { … }
bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
int64_t Offset1,
unsigned Size) const { … }
static bool isNoUnsignedWrap(MachineInstr *Addr) { … }
bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const { … }
bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const { … }
bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
Register Addr) const { … }
bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
unsigned ShAmtBits) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectMUBUFScratchOffset(
MachineOperand &Root) const { … }
std::pair<Register, unsigned>
AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
unsigned Size) const { … }
std::pair<Register, unsigned>
AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
unsigned Size) const { … }
std::pair<Register, int64_t>
AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
Register Root, const MachineRegisterInfo &MRI) const { … }
static void addZeroImm(MachineInstrBuilder &MIB) { … }
static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
uint32_t FormatLo, uint32_t FormatHi,
Register BasePtr) { … }
static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
const SIInstrInfo &TII, Register BasePtr) { … }
static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
const SIInstrInfo &TII, Register BasePtr) { … }
AMDGPUInstructionSelector::MUBUFAddressData
AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const { … }
bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const { … }
void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const { … }
bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
MachineOperand &Root, Register &VAddr, Register &RSrcReg,
Register &SOffset, int64_t &Offset) const { … }
bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
MachineOperand &Root, Register &RSrcReg, Register &SOffset,
int64_t &Offset) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const { … }
static std::optional<uint64_t>
getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const { … }
std::pair<Register, unsigned>
AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
bool &Matched) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
MachineOperand &Root) const { … }
InstructionSelector::ComplexRendererFns
AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const { … }
bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
MachineInstr &I, Intrinsic::ID IntrID) const { … }
unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) { … }
bool AMDGPUInstructionSelector::selectNamedBarrierInst(
MachineInstr &I, Intrinsic::ID IntrID) const { … }
bool AMDGPUInstructionSelector::selectSBarrierLeave(MachineInstr &I) const { … }
void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const { … }
void AMDGPUInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const { … }
bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const { … }
bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const { … }