NVPTXISelDAGToDAG.cpp | Explore in Territory

//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines an instruction selector for the NVPTX target.
//
//===----------------------------------------------------------------------===//

#include "NVPTXISelDAGToDAG.h"
#include "MCTargetDesc/NVPTXBaseInfo.h"
#include "NVPTXUtilities.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetIntrinsicInfo.h"

usingnamespacellvm;

#define DEBUG_TYPE …
#define PASS_NAME …

static cl::opt<bool>
    EnableRsqrtOpt("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden,
                   cl::desc("Enable reciprocal sqrt optimization"));

/// createNVPTXISelDag - This pass converts a legalized DAG into a
/// NVPTX-specific DAG, ready for instruction scheduling.
FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
                                       llvm::CodeGenOptLevel OptLevel) { … }

NVPTXDAGToDAGISelLegacy::NVPTXDAGToDAGISelLegacy(NVPTXTargetMachine &tm,
                                                 CodeGenOptLevel OptLevel)
    : … { … }

char NVPTXDAGToDAGISelLegacy::ID = …;

INITIALIZE_PASS(…)

NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
                                     CodeGenOptLevel OptLevel)
    : … { … }

bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { … }

int NVPTXDAGToDAGISel::getDivF32Level() const { … }

bool NVPTXDAGToDAGISel::usePrecSqrtF32() const { … }

bool NVPTXDAGToDAGISel::useF32FTZ() const { … }

bool NVPTXDAGToDAGISel::allowFMA() const { … }

bool NVPTXDAGToDAGISel::allowUnsafeFPMath() const { … }

bool NVPTXDAGToDAGISel::doRsqrtOpt() const { … }

/// Select - Select instructions not customized! Used for
/// expanded, promoted and normal instructions.
void NVPTXDAGToDAGISel::Select(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryIntrinsicChain(SDNode *N) { … }

// There's no way to specify FP16 and BF16 immediates in .(b)f16 ops, so we
// have to load them into an .(b)f16 register first.
bool NVPTXDAGToDAGISel::tryConstantFP(SDNode *N) { … }

// Map ISD:CONDCODE value to appropriate CmpMode expected by
// NVPTXInstPrinter::printCmpMode()
static unsigned getPTXCmpMode(const CondCodeSDNode &CondCode, bool FTZ) { … }

bool NVPTXDAGToDAGISel::SelectSETP_F16X2(SDNode *N) { … }

bool NVPTXDAGToDAGISel::SelectSETP_BF16X2(SDNode *N) { … }

// Find all instances of extract_vector_elt that use this v2f16 vector
// and coalesce them into a scattering move instruction.
bool NVPTXDAGToDAGISel::tryEXTRACT_VECTOR_ELEMENT(SDNode *N) { … }

static unsigned int getCodeAddrSpace(MemSDNode *N) { … }

namespace {

struct OperationOrderings { … };

static OperationOrderings
getOperationOrderings(MemSDNode *N, const NVPTXSubtarget *Subtarget) { … }

} // namespace

NVPTX::Scope NVPTXDAGToDAGISel::getOperationScope(MemSDNode *N,
                                                  NVPTX::Ordering O) const { … }

static bool canLowerToLDG(MemSDNode *N, const NVPTXSubtarget &Subtarget,
                          unsigned CodeAddrSpace, MachineFunction *F) { … }

static unsigned int getFenceOp(NVPTX::Ordering O, NVPTX::Scope S,
                               NVPTXSubtarget const *T) { … }

// Returns Memory Order and Scope of a memory instruction, and
// inserts any fence before the instruction that's required to
// implement its memory ordering.
std::pair<NVPTX::Ordering, NVPTX::Scope>
NVPTXDAGToDAGISel::insertMemoryInstructionFence(SDLoc DL, SDValue &Chain,
                                                MemSDNode *N) { … }

bool NVPTXDAGToDAGISel::tryIntrinsicNoChain(SDNode *N) { … }

void NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) { … }

void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) { … }

// Helper function template to reduce amount of boilerplate code for
// opcode selection.
static std::optional<unsigned>
pickOpcodeForVT(MVT::SimpleValueType VT, unsigned Opcode_i8,
                unsigned Opcode_i16, unsigned Opcode_i32,
                std::optional<unsigned> Opcode_i64, unsigned Opcode_f32,
                std::optional<unsigned> Opcode_f64) { … }

static int getLdStRegType(EVT VT) { … }

bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryStore(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryLoadParam(SDNode *Node) { … }

bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) { … }

// Helpers for constructing opcode (ex: NVPTX::StoreParamV4F32_iiri)
#define getOpcV2H(ty, opKind0, opKind1) …

#define getOpcV2H1(ty, opKind0, isImm1) …

#define getOpcodeForVectorStParamV2(ty, isimm) …

#define getOpcV4H(ty, opKind0, opKind1, opKind2, opKind3) …

#define getOpcV4H3(ty, opKind0, opKind1, opKind2, isImm3) …

#define getOpcV4H2(ty, opKind0, opKind1, isImm2, isImm3) …

#define getOpcV4H1(ty, opKind0, isImm1, isImm2, isImm3) …

#define getOpcodeForVectorStParamV4(ty, isimm) …

#define getOpcodeForVectorStParam(n, ty, isimm) …

static unsigned pickOpcodeForVectorStParam(SmallVector<SDValue, 8> &Ops,
                                           unsigned NumElts,
                                           MVT::SimpleValueType MemTy,
                                           SelectionDAG *CurDAG, SDLoc DL) { … }

bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) { … }

bool NVPTXDAGToDAGISel::tryTextureIntrinsic(SDNode *N) { … }

bool NVPTXDAGToDAGISel::trySurfaceIntrinsic(SDNode *N) { … }


/// SelectBFE - Look for instruction sequences that can be made more efficient
/// by using the 'bfe' (bit-field extract) PTX instruction
bool NVPTXDAGToDAGISel::tryBFE(SDNode *N) { … }

// SelectDirectAddr - Match a direct address for DAG.
// A direct address could be a globaladdress or externalsymbol.
bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { … }

// symbol+offset
bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
    SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { … }

// symbol+offset
bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
                                     SDValue &Base, SDValue &Offset) { … }

// symbol+offset
bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
                                       SDValue &Base, SDValue &Offset) { … }

// register+offset
bool NVPTXDAGToDAGISel::SelectADDRri_imp(
    SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { … }

// register+offset
bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
                                     SDValue &Base, SDValue &Offset) { … }

// register+offset
bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
                                       SDValue &Base, SDValue &Offset) { … }

bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
                                                 unsigned int spN) const { … }

/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
/// inline asm expressions.
bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
    const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
    std::vector<SDValue> &OutOps) { … }

void NVPTXDAGToDAGISel::SelectV2I64toI128(SDNode *N) { … }

void NVPTXDAGToDAGISel::SelectI128toV2I64(SDNode *N) { … }

/// GetConvertOpcode - Returns the CVT_ instruction opcode that implements a
/// conversion from \p SrcTy to \p DestTy.
unsigned NVPTXDAGToDAGISel::GetConvertOpcode(MVT DestTy, MVT SrcTy,
                                             LoadSDNode *LdNode) { … }

bool NVPTXDAGToDAGISel::tryFence(SDNode *N) { … }

NVPTXScopes::NVPTXScopes(LLVMContext &C) { … }

NVPTX::Scope NVPTXScopes::operator[](SyncScope::ID ID) const { … }

bool NVPTXScopes::empty() const { … }
llvm/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
