llvm/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp

//===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file This pass tries to apply several peephole SDWA patterns.
///
/// E.g. original:
///   V_LSHRREV_B32_e32 %0, 16, %1
///   V_ADD_CO_U32_e32 %2, %0, %3
///   V_LSHLREV_B32_e32 %4, 16, %2
///
/// Replace:
///   V_ADD_CO_U32_sdwa %4, %1, %3
///       dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
///
//===----------------------------------------------------------------------===//

#include "SIPeepholeSDWA.h"
#include "AMDGPU.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include <optional>

usingnamespacellvm;

#define DEBUG_TYPE …

STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
STATISTIC(NumSDWAInstructionsPeepholed,
          "Number of instruction converted to SDWA.");

namespace {

bool isConvertibleToSDWA(MachineInstr &MI, const GCNSubtarget &ST,
                         const SIInstrInfo *TII);
class SDWAOperand;
class SDWADstOperand;

SDWAOperandsVector;
SDWAOperandsMap;

class SIPeepholeSDWA { … };

class SIPeepholeSDWALegacy : public MachineFunctionPass { … };

class SDWAOperand { … };

usingnamespaceAMDGPU::SDWA;

class SDWASrcOperand : public SDWAOperand { … };

class SDWADstOperand : public SDWAOperand { … };

class SDWADstPreserveOperand : public SDWADstOperand { … };

} // end anonymous namespace

INITIALIZE_PASS(…)

char SIPeepholeSDWALegacy::ID = …;

char &llvm::SIPeepholeSDWALegacyID = …;

FunctionPass *llvm::createSIPeepholeSDWALegacyPass() { … }

#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
static raw_ostream& operator<<(raw_ostream &OS, SdwaSel Sel) {
  switch(Sel) {
  case BYTE_0: OS << "BYTE_0"; break;
  case BYTE_1: OS << "BYTE_1"; break;
  case BYTE_2: OS << "BYTE_2"; break;
  case BYTE_3: OS << "BYTE_3"; break;
  case WORD_0: OS << "WORD_0"; break;
  case WORD_1: OS << "WORD_1"; break;
  case DWORD:  OS << "DWORD"; break;
  }
  return OS;
}

static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
  switch(Un) {
  case UNUSED_PAD: OS << "UNUSED_PAD"; break;
  case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
  case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
  }
  return OS;
}

LLVM_DUMP_METHOD
void SDWASrcOperand::print(raw_ostream& OS) const {
  OS << "SDWA src: " << *getTargetOperand()
    << " src_sel:" << getSrcSel()
    << " abs:" << getAbs() << " neg:" << getNeg()
    << " sext:" << getSext() << '\n';
}

LLVM_DUMP_METHOD
void SDWADstOperand::print(raw_ostream& OS) const {
  OS << "SDWA dst: " << *getTargetOperand()
    << " dst_sel:" << getDstSel()
    << " dst_unused:" << getDstUnused() << '\n';
}

LLVM_DUMP_METHOD
void SDWADstPreserveOperand::print(raw_ostream& OS) const {
  OS << "SDWA preserve dst: " << *getTargetOperand()
    << " dst_sel:" << getDstSel()
    << " preserve:" << *getPreservedOperand() << '\n';
}

#endif

static void copyRegOperand(MachineOperand &To, const MachineOperand &From) { … }

static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) { … }

static MachineOperand *findSingleRegUse(const MachineOperand *Reg,
                                        const MachineRegisterInfo *MRI) { … }

static MachineOperand *findSingleRegDef(const MachineOperand *Reg,
                                        const MachineRegisterInfo *MRI) { … }

uint64_t SDWASrcOperand::getSrcMods(const SIInstrInfo *TII,
                                    const MachineOperand *SrcOp) const { … }

MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII,
                                                 const GCNSubtarget &ST,
                                                 SDWAOperandsMap *PotentialMatches) { … }

bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) { … }

MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII,
                                                 const GCNSubtarget &ST,
                                                 SDWAOperandsMap *PotentialMatches) { … }

bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) { … }

bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI,
                                           const SIInstrInfo *TII) { … }

std::optional<int64_t>
SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const { … }

std::unique_ptr<SDWAOperand>
SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) { … }

#if !defined(NDEBUG)
static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) {
  Operand.print(OS);
  return OS;
}
#endif

void SIPeepholeSDWA::matchSDWAOperands(MachineBasicBlock &MBB) { … }

// Convert the V_ADD_CO_U32_e64 into V_ADD_CO_U32_e32. This allows
// isConvertibleToSDWA to perform its transformation on V_ADD_CO_U32_e32 into
// V_ADD_CO_U32_sdwa.
//
// We are transforming from a VOP3 into a VOP2 form of the instruction.
//   %19:vgpr_32 = V_AND_B32_e32 255,
//       killed %16:vgpr_32, implicit $exec
//   %47:vgpr_32, %49:sreg_64_xexec = V_ADD_CO_U32_e64
//       %26.sub0:vreg_64, %19:vgpr_32, implicit $exec
//  %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
//       %26.sub1:vreg_64, %54:vgpr_32, killed %49:sreg_64_xexec, implicit $exec
//
// becomes
//   %47:vgpr_32 = V_ADD_CO_U32_sdwa
//       0, %26.sub0:vreg_64, 0, killed %16:vgpr_32, 0, 6, 0, 6, 0,
//       implicit-def $vcc, implicit $exec
//  %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
//       %26.sub1:vreg_64, %54:vgpr_32, killed $vcc, implicit $exec
void SIPeepholeSDWA::pseudoOpConvertToVOP2(MachineInstr &MI,
                                           const GCNSubtarget &ST) const { … }

namespace {
bool isConvertibleToSDWA(MachineInstr &MI,
                         const GCNSubtarget &ST,
                         const SIInstrInfo* TII) { … }
} // namespace

bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
                                   const SDWAOperandsVector &SDWAOperands) { … }

// If an instruction was converted to SDWA it should not have immediates or SGPR
// operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
                                            const GCNSubtarget &ST) const { … }

bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) { … }

bool SIPeepholeSDWA::run(MachineFunction &MF) { … }

PreservedAnalyses SIPeepholeSDWAPass::run(MachineFunction &MF,
                                          MachineFunctionAnalysisManager &) { … }