GCNHazardRecognizer.cpp | Explore in Territory

//===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements hazard recognizers for scheduling on GCN processors.
//
//===----------------------------------------------------------------------===//

#include "GCNHazardRecognizer.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/TargetParser/TargetParser.h"

usingnamespacellvm;

namespace {

struct MFMAPaddingRatioParser : public cl::parser<unsigned> { … };

} // end anonymous namespace

static cl::opt<unsigned, false, MFMAPaddingRatioParser>
    MFMAPaddingRatio("amdgpu-mfma-padding-ratio", cl::init(0), cl::Hidden,
                     cl::desc("Fill a percentage of the latency between "
                              "neighboring MFMA with s_nops."));

static cl::opt<unsigned> MaxExhaustiveHazardSearch(
    "amdgpu-max-exhaustive-hazard-search", cl::init(128), cl::Hidden,
    cl::desc("Maximum function size for exhausive hazard search"));

//===----------------------------------------------------------------------===//
// Hazard Recognizer Implementation
//===----------------------------------------------------------------------===//

static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
                                                 const GCNSubtarget &ST);

GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF)
    : … { … }

void GCNHazardRecognizer::Reset() { … }

void GCNHazardRecognizer::EmitInstruction(SUnit *SU) { … }

void GCNHazardRecognizer::EmitInstruction(MachineInstr *MI) { … }

static bool isDivFMas(unsigned Opcode) { … }

static bool isSGetReg(unsigned Opcode) { … }

static bool isSSetReg(unsigned Opcode) { … }

static bool isRWLane(unsigned Opcode) { … }

static bool isRFE(unsigned Opcode) { … }

static bool isSMovRel(unsigned Opcode) { … }

static bool isDGEMM(unsigned Opcode) { … }

static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) { … }

static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
                                    const MachineInstr &MI) { … }

static bool isPermlane(const MachineInstr &MI) { … }

static bool isLdsDma(const MachineInstr &MI) { … }

static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) { … }

ScheduleHazardRecognizer::HazardType
GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { … }

static void insertNoopsInBundle(MachineInstr *MI, const SIInstrInfo &TII,
                                unsigned Quantity) { … }

unsigned
GCNHazardRecognizer::getMFMAPipelineWaitStates(const MachineInstr &MI) const { … }

void GCNHazardRecognizer::processBundle() { … }

void GCNHazardRecognizer::runOnInstruction(MachineInstr *MI) { … }

unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) { … }

unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { … }

void GCNHazardRecognizer::EmitNoop() { … }

void GCNHazardRecognizer::AdvanceCycle() { … }

void GCNHazardRecognizer::RecedeCycle() { … }

//===----------------------------------------------------------------------===//
// Helper Functions
//===----------------------------------------------------------------------===//

using HazardFnResult = enum { … };

IsExpiredFn;
GetNumWaitStatesFn;

// Search for a hazard in a block and its predecessors.
template <typename StateT>
static bool
hasHazard(StateT State,
          function_ref<HazardFnResult(StateT &, const MachineInstr &)> IsHazard,
          function_ref<void(StateT &, const MachineInstr &)> UpdateState,
          const MachineBasicBlock *MBB,
          MachineBasicBlock::const_reverse_instr_iterator I,
          DenseSet<const MachineBasicBlock *> &Visited) { … }

// Returns a minimum wait states since \p I walking all predecessors.
// Only scans until \p IsExpired does not return true.
// Can only be run in a hazard recognizer mode.
static int getWaitStatesSince(
    GCNHazardRecognizer::IsHazardFn IsHazard, const MachineBasicBlock *MBB,
    MachineBasicBlock::const_reverse_instr_iterator I, int WaitStates,
    IsExpiredFn IsExpired, DenseSet<const MachineBasicBlock *> &Visited,
    GetNumWaitStatesFn GetNumWaitStates = SIInstrInfo::getNumWaitStates) { … }

static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
                              const MachineInstr *MI, IsExpiredFn IsExpired) { … }

int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) { … }

int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
                                               IsHazardFn IsHazardDef,
                                               int Limit) { … }

int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
                                                  int Limit) { … }

//===----------------------------------------------------------------------===//
// No-op Hazard Detection
//===----------------------------------------------------------------------===//

static void addRegUnits(const SIRegisterInfo &TRI, BitVector &BV,
                        MCRegister Reg) { … }

static void addRegsToSet(const SIRegisterInfo &TRI,
                         iterator_range<MachineInstr::const_mop_iterator> Ops,
                         BitVector &DefSet, BitVector &UseSet) { … }

void GCNHazardRecognizer::addClauseInst(const MachineInstr &MI) { … }

static bool breaksSMEMSoftClause(MachineInstr *MI) { … }

static bool breaksVMEMSoftClause(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkSoftClauseHazards(MachineInstr *MEM) { … }

int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) { … }

int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) { … }

int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) { … }

int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) { … }

int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) { … }

int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) { … }

int GCNHazardRecognizer::createsVALUHazard(const MachineInstr &MI) { … }

int
GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
                                            const MachineRegisterInfo &MRI) { … }

/// Dest sel forwarding issue occurs if additional logic is needed to swizzle /
/// pack the computed value into correct bit position of the dest register. This
/// occurs if we have SDWA with dst_sel != DWORD or if we have op_sel with
/// dst_sel that is not aligned to the register. This function analayzes the \p
/// MI and \returns an operand with dst forwarding issue, or nullptr if
/// none exists.
static const MachineOperand *
getDstSelForwardingOperand(const MachineInstr &MI, const GCNSubtarget &ST) { … }

/// Checks whether the provided \p MI "consumes" the operand with a Dest sel
/// fowarding issue \p Dst . We may "consume" the Dst via a standard explicit
/// RAW, or through irregular ways (e.g implicit RAW, certain types of WAW)
static bool consumesDstSelForwardingOperand(const MachineInstr *VALU,
                                            const MachineOperand *Dst,
                                            const SIRegisterInfo *TRI) { … }

int GCNHazardRecognizer::checkVALUHazards(MachineInstr *VALU) { … }

int GCNHazardRecognizer::checkInlineAsmHazards(MachineInstr *IA) { … }

int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) { … }

int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) { … }

int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) { … }

void GCNHazardRecognizer::fixHazards(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixVcmpxPermlaneHazards(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixVMEMtoScalarWriteHazards(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixVcmpxExecWARHazard(MachineInstr *MI) { … }

static bool shouldRunLdsBranchVmemWARHazardFixup(const MachineFunction &MF,
                                                 const GCNSubtarget &ST) { … }

static bool isStoreCountWaitZero(const MachineInstr &I) { … }

bool GCNHazardRecognizer::fixLdsBranchVmemWARHazard(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixLdsDirectVALUHazard(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixLdsDirectVMEMHazard(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixWMMAHazards(MachineInstr *MI) { … }

bool GCNHazardRecognizer::fixShift64HighRegBug(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkNSAtoVMEMHazard(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkMAIHazards(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkMFMAPadding(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkMAIHazards908(MachineInstr *MI) { … }

static int
GFX940_XDL_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) { … }

static int
GFX940_SMFMA_N_PassWritesVGPROverlappedSMFMASrcCWaitStates(int NumPasses) { … }

static int
GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { … }

static int GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(int NumPasses) { … }

int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) { … }

int GCNHazardRecognizer::checkMAILdStHazards(MachineInstr *MI) { … }

static int GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { … }

static int GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(int NumPasses) { … }

static int GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { … }

static int GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(int NumPasses) { … }

int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) { … }

bool GCNHazardRecognizer::ShouldPreferAnother(SUnit *SU) { … }

// Adjust global offsets for instructions bundled with S_GETPC_B64 after
// insertion of a new instruction.
static void updateGetPCBundle(MachineInstr *NewMI) { … }

bool GCNHazardRecognizer::fixVALUMaskWriteHazard(MachineInstr *MI) { … }

// Return the numeric ID 0-63 of an 64b SGPR pair for a given SGPR.
// i.e. SGPR0 = SGPR0_SGPR1 = 0, SGPR3 = SGPR2_SGPR3 = 1, etc
static std::optional<unsigned> sgprPairNumber(Register Reg,
                                              const SIRegisterInfo &TRI) { … }

// For VALUReadSGPRHazard: pre-compute a bit vector of all SGPRs used by VALUs.
void GCNHazardRecognizer::computeVALUHazardSGPRs(MachineFunction *MMF) { … }

bool GCNHazardRecognizer::fixVALUReadSGPRHazard(MachineInstr *MI) { … }

static bool ensureEntrySetPrio(MachineFunction *MF, int Priority,
                               const SIInstrInfo &TII) { … }

bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) { … }
llvm/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
