llvm/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This implements the SelectionDAG class.
//
//===----------------------------------------------------------------------===//

#include "llvm/CodeGen/SelectionDAG.h"
#include "SDNodeDbgValue.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/APSInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/FoldingSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/RuntimeLibcallUtil.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/Mutex.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <deque>
#include <limits>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>

usingnamespacellvm;
usingnamespacellvm::SDPatternMatch;

/// makeVTList - Return an instance of the SDVTList struct initialized with the
/// specified members.
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { … }

// Default null implementations of the callbacks.
void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) { … }
void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) { … }
void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) { … }

void SelectionDAG::DAGNodeDeletedListener::anchor() { … }
void SelectionDAG::DAGNodeInsertedListener::anchor() { … }

#define DEBUG_TYPE …

static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
       cl::Hidden, cl::init(true),
       cl::desc("Gang up loads and stores generated by inlining of memcpy"));

static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
       cl::desc("Number limit for gluing ld/st of memcpy."),
       cl::Hidden, cl::init(0));

static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { … }

//===----------------------------------------------------------------------===//
//                              ConstantFPSDNode Class
//===----------------------------------------------------------------------===//

/// isExactlyValue - We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.
bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { … }

bool ConstantFPSDNode::isValueValidForType(EVT VT,
                                           const APFloat& Val) { … }

//===----------------------------------------------------------------------===//
//                              ISD Namespace
//===----------------------------------------------------------------------===//

bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { … }

// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
// specializations of the more general isConstantSplatVector()?

bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { … }

bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { … }

bool ISD::isBuildVectorAllOnes(const SDNode *N) { … }

bool ISD::isBuildVectorAllZeros(const SDNode *N) { … }

bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { … }

bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { … }

bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
                             bool Signed) { … }

bool ISD::allOperandsUndef(const SDNode *N) { … }

bool ISD::isFreezeUndef(const SDNode *N) { … }

template <typename ConstNodeType>
bool ISD::matchUnaryPredicateImpl(SDValue Op,
                                  std::function<bool(ConstNodeType *)> Match,
                                  bool AllowUndefs) { … }
// Build used template types.
template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
    SDValue, std::function<bool(ConstantSDNode *)>, bool);
template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
    SDValue, std::function<bool(ConstantFPSDNode *)>, bool);

bool ISD::matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs, bool AllowTypeMismatch) { … }

ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { … }

bool ISD::isVPOpcode(unsigned Opcode) { … }

bool ISD::isVPBinaryOp(unsigned Opcode) { … }

bool ISD::isVPReduction(unsigned Opcode) { … }

/// The operand position of the vector mask.
std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { … }

/// The operand position of the explicit vector length parameter.
std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { … }

std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
                                                bool hasFPExcept) { … }

std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) { … }

ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { … }

ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { … }

static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { … }

ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { … }

ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
                                               bool isIntegerLike) { … }

/// For an integer comparison, return 1 if the comparison is a signed operation
/// and 2 if the result is an unsigned comparison. Return zero if the operation
/// does not depend on the sign of the input (setne and seteq).
static int isSignedOp(ISD::CondCode Opcode) { … }

ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
                                       EVT Type) { … }

ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
                                        EVT Type) { … }

//===----------------------------------------------------------------------===//
//                           SDNode Profile Support
//===----------------------------------------------------------------------===//

/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  { … }

/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
/// solely with their pointer.
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { … }

/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static void AddNodeIDOperands(FoldingSetNodeID &ID,
                              ArrayRef<SDValue> Ops) { … }

/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static void AddNodeIDOperands(FoldingSetNodeID &ID,
                              ArrayRef<SDUse> Ops) { … }

static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
                          SDVTList VTList, ArrayRef<SDValue> OpList) { … }

/// If this is an SDNode with special info, add this info to the NodeID data.
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { … }

/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
/// data.
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { … }

//===----------------------------------------------------------------------===//
//                              SelectionDAG Class
//===----------------------------------------------------------------------===//

/// doNotCSE - Return true if CSE should not be performed for this node.
static bool doNotCSE(SDNode *N) { … }

/// RemoveDeadNodes - This method deletes all unreachable nodes in the
/// SelectionDAG.
void SelectionDAG::RemoveDeadNodes() { … }

/// RemoveDeadNodes - This method deletes the unreachable nodes in the
/// given list, and any nodes that become unreachable as a result.
void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { … }

void SelectionDAG::RemoveDeadNode(SDNode *N){ … }

void SelectionDAG::DeleteNode(SDNode *N) { … }

void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { … }

void SDDbgInfo::add(SDDbgValue *V, bool isParameter) { … }

void SDDbgInfo::erase(const SDNode *Node) { … }

void SelectionDAG::DeallocateNode(SDNode *N) { … }

#ifndef NDEBUG
/// VerifySDNode - Check the given SDNode.  Aborts if it is invalid.
static void VerifySDNode(SDNode *N, const TargetLowering *TLI) {
  switch (N->getOpcode()) {
  default:
    if (N->getOpcode() > ISD::BUILTIN_OP_END)
      TLI->verifyTargetSDNode(N);
    break;
  case ISD::BUILD_PAIR: {
    EVT VT = N->getValueType(0);
    assert(N->getNumValues() == 1 && "Too many results!");
    assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
           "Wrong return type!");
    assert(N->getNumOperands() == 2 && "Wrong number of operands!");
    assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
           "Mismatched operand types!");
    assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
           "Wrong operand type!");
    assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
           "Wrong return type size");
    break;
  }
  case ISD::BUILD_VECTOR: {
    assert(N->getNumValues() == 1 && "Too many results!");
    assert(N->getValueType(0).isVector() && "Wrong return type!");
    assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
           "Wrong number of operands!");
    EVT EltVT = N->getValueType(0).getVectorElementType();
    for (const SDUse &Op : N->ops()) {
      assert((Op.getValueType() == EltVT ||
              (EltVT.isInteger() && Op.getValueType().isInteger() &&
               EltVT.bitsLE(Op.getValueType()))) &&
             "Wrong operand type!");
      assert(Op.getValueType() == N->getOperand(0).getValueType() &&
             "Operands must all have the same type");
    }
    break;
  }
  }
}
#endif // NDEBUG

/// Insert a newly allocated node into the DAG.
///
/// Handles insertion into the all nodes list and CSE map, as well as
/// verification and other common operations when a new node is allocated.
void SelectionDAG::InsertNode(SDNode *N) { … }

/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
/// correspond to it.  This is useful when we're about to delete or repurpose
/// the node.  We don't want future request for structurally identical nodes
/// to return N anymore.
bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { … }

/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
/// maps and modified in place. Add it back to the CSE maps, unless an identical
/// node already exists, in which case transfer all its users to the existing
/// node. This transfer can potentially trigger recursive merging.
void
SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { … }

/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
/// were replaced with those specified.  If this node is never memoized,
/// return null, otherwise return a pointer to the slot it would take.  If a
/// node already exists with these operands, the slot will be non-null.
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
                                           void *&InsertPos) { … }

/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
/// were replaced with those specified.  If this node is never memoized,
/// return null, otherwise return a pointer to the slot it would take.  If a
/// node already exists with these operands, the slot will be non-null.
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
                                           SDValue Op1, SDValue Op2,
                                           void *&InsertPos) { … }

/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
/// were replaced with those specified.  If this node is never memoized,
/// return null, otherwise return a pointer to the slot it would take.  If a
/// node already exists with these operands, the slot will be non-null.
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
                                           void *&InsertPos) { … }

Align SelectionDAG::getEVTAlign(EVT VT) const { … }

// EntryNode could meaningfully have debug info if we can find it...
SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
    : … { … }

void SelectionDAG::init(MachineFunction &NewMF,
                        OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
                        const TargetLibraryInfo *LibraryInfo,
                        UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
                        BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
                        FunctionVarLocs const *VarLocs) { … }

SelectionDAG::~SelectionDAG() { … }

bool SelectionDAG::shouldOptForSize() const { … }

void SelectionDAG::allnodes_clear() { … }

SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
                                          void *&InsertPos) { … }

SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
                                          const SDLoc &DL, void *&InsertPos) { … }

void SelectionDAG::clear() { … }

SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { … }

std::pair<SDValue, SDValue>
SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
                                       const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
                                                EVT VT) { … }

SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
                                               EVT VT) { … }

SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
                                               EVT VT) { … }

SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
                                        EVT OpVT) { … }

SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
                                           SDValue EVL, const SDLoc &DL,
                                           EVT VT) { … }

SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) { … }

/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { … }

SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { … }

SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
                                      SDValue Mask, SDValue EVL, EVT VT) { … }

SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
                                         SDValue Mask, SDValue EVL) { … }

SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
                                       SDValue Mask, SDValue EVL) { … }

SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
                                      EVT OpVT) { … }

SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
                                  bool isT, bool isO) { … }

SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
                                  bool isT, bool isO) { … }

SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
                                  EVT VT, bool isT, bool isO) { … }

SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
                                        bool isT, bool isO) { … }

SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
                                         bool IsOpaque) { … }

SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
                                        bool isTarget) { … }

SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
                                             const SDLoc &DL) { … }

SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
                                             const SDLoc &DL) { … }

SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
                                           bool isTarget) { … }

SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
                                    bool isTarget) { … }

SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
                                    EVT VT, bool isTarget) { … }

SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
                                    bool isTarget) { … }

SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
                                       EVT VT, int64_t Offset, bool isTargetGA,
                                       unsigned TargetFlags) { … }

SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { … }

SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
                                   unsigned TargetFlags) { … }

SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
                                            const SDLoc &DL) { … }

SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
                                      MaybeAlign Alignment, int Offset,
                                      bool isTarget, unsigned TargetFlags) { … }

SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
                                      MaybeAlign Alignment, int Offset,
                                      bool isTarget, unsigned TargetFlags) { … }

SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { … }

SDValue SelectionDAG::getValueType(EVT VT) { … }

SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { … }

SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { … }

SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
                                              unsigned TargetFlags) { … }

SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { … }

SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
                                bool ConstantFold) { … }

SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC,
                                      bool ConstantFold) { … }

SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) { … }

SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
                                    const APInt &StepVal) { … }

/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
/// point at N1 to point at N2 and indices that point at N2 to point at N1.
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) { … }

SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
                                       SDValue N2, ArrayRef<int> Mask) { … }

SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { … }

SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { … }

SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { … }

SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
                                 MCSymbol *Label) { … }

SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
                                   SDValue Root, MCSymbol *Label) { … }

SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
                                      int64_t Offset, bool isTarget,
                                      unsigned TargetFlags) { … }

SDValue SelectionDAG::getSrcValue(const Value *V) { … }

SDValue SelectionDAG::getMDNode(const MDNode *MD) { … }

SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { … }

SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
                                       unsigned SrcAS, unsigned DestAS) { … }

SDValue SelectionDAG::getFreeze(SDValue V) { … }

/// getShiftAmountOperand - Return the specified value casted to
/// the target's desired shift amount type.
SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { … }

SDValue SelectionDAG::getPartialReduceAdd(SDLoc DL, EVT ReducedTy, SDValue Op1,
                                          SDValue Op2) { … }

SDValue SelectionDAG::expandVAArg(SDNode *Node) { … }

SDValue SelectionDAG::expandVACopy(SDNode *Node) { … }

Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { … }

SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { … }

SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { … }

SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { … }

SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
                                ISD::CondCode Cond, const SDLoc &dl) { … }

/// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
/// use this predicate to simplify operations downstream.
bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { … }

/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
/// this predicate to simplify operations downstream.  Mask is known to be zero
/// for bits that V cannot have.
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
                                     unsigned Depth) const { … }

/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
/// DemandedElts.  We use this predicate to simplify operations downstream.
/// Mask is known to be zero for bits that V cannot have.
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
                                     const APInt &DemandedElts,
                                     unsigned Depth) const { … }

/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
/// DemandedElts.  We use this predicate to simplify operations downstream.
bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
                                      unsigned Depth /* = 0 */) const { … }

/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
                                        unsigned Depth) const { … }

APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
                                                   const APInt &DemandedElts,
                                                   unsigned Depth) const { … }

/// isSplatValue - Return true if the vector V has the same value
/// across all DemandedElts. For scalable vectors, we don't know the
/// number of lanes at compile time.  Instead, we use a 1 bit APInt
/// to represent a conservative value for all lanes; that is, that
/// one bit value is implicitly splatted across all lanes.
bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
                                APInt &UndefElts, unsigned Depth) const { … }

/// Helper wrapper to main isSplatValue function.
bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const { … }

SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { … }

SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) { … }

std::optional<ConstantRange>
SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
                                       unsigned Depth) const { … }

std::optional<uint64_t>
SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
                                  unsigned Depth) const { … }

std::optional<uint64_t>
SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const { … }

std::optional<uint64_t>
SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
                                         unsigned Depth) const { … }

std::optional<uint64_t>
SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const { … }

std::optional<uint64_t>
SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
                                         unsigned Depth) const { … }

std::optional<uint64_t>
SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const { … }

/// Determine which bits of Op are known to be either zero or one and return
/// them in Known. For vectors, the known bits are those that are shared by
/// every vector element.
KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const { … }

/// Determine which bits of Op are known to be either zero or one and return
/// them in Known. The DemandedElts argument allows us to only collect the known
/// bits that are shared by the requested vector elements.
KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
                                         unsigned Depth) const { … }

/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) { … }

SelectionDAG::OverflowKind
SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const { … }

SelectionDAG::OverflowKind
SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const { … }

SelectionDAG::OverflowKind
SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const { … }

SelectionDAG::OverflowKind
SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const { … }

SelectionDAG::OverflowKind
SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const { … }

SelectionDAG::OverflowKind
SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const { … }

bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const { … }

bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const { … }

unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { … }

unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
                                          unsigned Depth) const { … }

unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
                                                 unsigned Depth) const { … }

unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
                                                 const APInt &DemandedElts,
                                                 unsigned Depth) const { … }

bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
                                                    unsigned Depth) const { … }

bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
                                                    const APInt &DemandedElts,
                                                    bool PoisonOnly,
                                                    unsigned Depth) const { … }

bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
                                          bool ConsiderFlags,
                                          unsigned Depth) const { … }

bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
                                          bool PoisonOnly, bool ConsiderFlags,
                                          unsigned Depth) const { … }

bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const { … }

bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { … }

bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const { … }

bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const { … }

bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const { … }

bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const { … }

bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { … }

// Only bits set in Mask must be negated, other bits may be arbitrary.
SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) { … }

static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) { … }

// FIXME: unify with llvm::haveNoCommonBitsSet.
bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { … }

static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
                               SelectionDAG &DAG) { … }

static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
                                ArrayRef<SDValue> Ops,
                                SelectionDAG &DAG) { … }

/// Try to simplify vector concatenation to an input value, undef, or build
/// vector.
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
                                  ArrayRef<SDValue> Ops,
                                  SelectionDAG &DAG) { … }

/// Gets or creates the specified node.
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, const SDNodeFlags Flags) { … }

static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
                                      const APInt &C2) { … }
// Handle constant folding with UNDEF.
// TODO: Handle more cases.
static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
                                               bool IsUndef1, const APInt &C2,
                                               bool IsUndef2) { … }

SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
                                       const GlobalAddressSDNode *GA,
                                       const SDNode *N2) { … }

bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { … }

SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
                                             EVT VT, ArrayRef<SDValue> Ops,
                                             SDNodeFlags Flags) { … }

SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
                                         EVT VT, ArrayRef<SDValue> Ops) { … }

SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2) { … }

void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
                                                SDValue &N2) const { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, SDValue N3) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, SDValue N3,
                              const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
                              const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
                              SDValue N5, const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
                              SDValue N5) { … }

/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
/// the incoming stack arguments to be loaded from the stack.
SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { … }

/// getMemsetValue - Vectorized representation of the memset value
/// operand.
static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
                              const SDLoc &dl) { … }

/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
/// used when a memcpy is turned into a memset when the source is a constant
/// string ptr.
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
                                  const TargetLowering &TLI,
                                  const ConstantDataArraySlice &Slice) { … }

SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
                                           const SDLoc &DL,
                                           const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
                                           const SDLoc &DL,
                                           const SDNodeFlags Flags) { … }

/// Returns true if memcpy source is constant data.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { … }

static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
                                      SelectionDAG &DAG) { … }

static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
                          SmallVector<SDValue, 32> &OutChains, unsigned From,
                          unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
                          SmallVector<SDValue, 16> &OutStoreChains) { … }

static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
                                       SDValue Chain, SDValue Dst, SDValue Src,
                                       uint64_t Size, Align Alignment,
                                       bool isVol, bool AlwaysInline,
                                       MachinePointerInfo DstPtrInfo,
                                       MachinePointerInfo SrcPtrInfo,
                                       const AAMDNodes &AAInfo, AAResults *AA) { … }

static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
                                        SDValue Chain, SDValue Dst, SDValue Src,
                                        uint64_t Size, Align Alignment,
                                        bool isVol, bool AlwaysInline,
                                        MachinePointerInfo DstPtrInfo,
                                        MachinePointerInfo SrcPtrInfo,
                                        const AAMDNodes &AAInfo) { … }

/// Lower the call to 'memset' intrinsic function into a series of store
/// operations.
///
/// \param DAG Selection DAG where lowered code is placed.
/// \param dl Link to corresponding IR location.
/// \param Chain Control flow dependency.
/// \param Dst Pointer to destination memory location.
/// \param Src Value of byte to write into the memory.
/// \param Size Number of bytes to write.
/// \param Alignment Alignment of the destination in bytes.
/// \param isVol True if destination is volatile.
/// \param AlwaysInline Makes sure no function call is generated.
/// \param DstPtrInfo IR information on the memory pointer.
/// \returns New head in the control flow, if lowering was successful, empty
/// SDValue otherwise.
///
/// The function tries to replace 'llvm.memset' intrinsic with several store
/// operations and value calculation code. This is usually profitable for small
/// memory size or when the semantic requires inlining.
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
                               SDValue Chain, SDValue Dst, SDValue Src,
                               uint64_t Size, Align Alignment, bool isVol,
                               bool AlwaysInline, MachinePointerInfo DstPtrInfo,
                               const AAMDNodes &AAInfo) { … }

static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
                                            unsigned AS) { … }

SDValue SelectionDAG::getMemcpy(
    SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
    Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
    std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
    MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) { … }

SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
                                      SDValue Dst, SDValue Src, SDValue Size,
                                      Type *SizeTy, unsigned ElemSz,
                                      bool isTailCall,
                                      MachinePointerInfo DstPtrInfo,
                                      MachinePointerInfo SrcPtrInfo) { … }

SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
                                 SDValue Src, SDValue Size, Align Alignment,
                                 bool isVol, const CallInst *CI,
                                 std::optional<bool> OverrideTailCall,
                                 MachinePointerInfo DstPtrInfo,
                                 MachinePointerInfo SrcPtrInfo,
                                 const AAMDNodes &AAInfo, AAResults *AA) { … }

SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
                                       SDValue Dst, SDValue Src, SDValue Size,
                                       Type *SizeTy, unsigned ElemSz,
                                       bool isTailCall,
                                       MachinePointerInfo DstPtrInfo,
                                       MachinePointerInfo SrcPtrInfo) { … }

SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
                                SDValue Src, SDValue Size, Align Alignment,
                                bool isVol, bool AlwaysInline,
                                const CallInst *CI,
                                MachinePointerInfo DstPtrInfo,
                                const AAMDNodes &AAInfo) { … }

SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
                                      SDValue Dst, SDValue Value, SDValue Size,
                                      Type *SizeTy, unsigned ElemSz,
                                      bool isTailCall,
                                      MachinePointerInfo DstPtrInfo) { … }

SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                                SDVTList VTList, ArrayRef<SDValue> Ops,
                                MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
                                       EVT MemVT, SDVTList VTs, SDValue Chain,
                                       SDValue Ptr, SDValue Cmp, SDValue Swp,
                                       MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                                SDValue Chain, SDValue Ptr, SDValue Val,
                                MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                                EVT VT, SDValue Chain, SDValue Ptr,
                                MachineMemOperand *MMO) { … }

/// getMergeValues - Create a MERGE_VALUES node from the given operands.
SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) { … }

SDValue SelectionDAG::getMemIntrinsicNode(
    unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
    EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
    MachineMemOperand::Flags Flags, LocationSize Size,
    const AAMDNodes &AAInfo) { … }

SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
                                          SDVTList VTList,
                                          ArrayRef<SDValue> Ops, EVT MemVT,
                                          MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
                                      SDValue Chain, int FrameIndex,
                                      int64_t Size, int64_t Offset) { … }

SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
                                         uint64_t Guid, uint64_t Index,
                                         uint32_t Attr) { … }

/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
/// MachinePointerInfo record from it.  This is particularly useful because the
/// code generator has many cases where it doesn't bother passing in a
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
                                           SelectionDAG &DAG, SDValue Ptr,
                                           int64_t Offset = 0) { … }

/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
/// MachinePointerInfo record from it.  This is particularly useful because the
/// code generator has many cases where it doesn't bother passing in a
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
                                           SelectionDAG &DAG, SDValue Ptr,
                                           SDValue OffsetOp) { … }

SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                              EVT VT, const SDLoc &dl, SDValue Chain,
                              SDValue Ptr, SDValue Offset,
                              MachinePointerInfo PtrInfo, EVT MemVT,
                              Align Alignment,
                              MachineMemOperand::Flags MMOFlags,
                              const AAMDNodes &AAInfo, const MDNode *Ranges) { … }

SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                              EVT VT, const SDLoc &dl, SDValue Chain,
                              SDValue Ptr, SDValue Offset, EVT MemVT,
                              MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
                              SDValue Ptr, MachinePointerInfo PtrInfo,
                              MaybeAlign Alignment,
                              MachineMemOperand::Flags MMOFlags,
                              const AAMDNodes &AAInfo, const MDNode *Ranges) { … }

SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
                              SDValue Ptr, MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
                                 EVT VT, SDValue Chain, SDValue Ptr,
                                 MachinePointerInfo PtrInfo, EVT MemVT,
                                 MaybeAlign Alignment,
                                 MachineMemOperand::Flags MMOFlags,
                                 const AAMDNodes &AAInfo) { … }

SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
                                 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
                                 MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
                                     SDValue Base, SDValue Offset,
                                     ISD::MemIndexedMode AM) { … }

SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                               SDValue Ptr, MachinePointerInfo PtrInfo,
                               Align Alignment,
                               MachineMemOperand::Flags MMOFlags,
                               const AAMDNodes &AAInfo) { … }

SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                               SDValue Ptr, MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                                    SDValue Ptr, MachinePointerInfo PtrInfo,
                                    EVT SVT, Align Alignment,
                                    MachineMemOperand::Flags MMOFlags,
                                    const AAMDNodes &AAInfo) { … }

SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                                    SDValue Ptr, EVT SVT,
                                    MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
                                      SDValue Base, SDValue Offset,
                                      ISD::MemIndexedMode AM) { … }

SDValue SelectionDAG::getLoadVP(
    ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
    SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
    MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
    MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
    const MDNode *Ranges, bool IsExpanding) { … }

SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
                                ISD::LoadExtType ExtType, EVT VT,
                                const SDLoc &dl, SDValue Chain, SDValue Ptr,
                                SDValue Offset, SDValue Mask, SDValue EVL,
                                EVT MemVT, MachineMemOperand *MMO,
                                bool IsExpanding) { … }

SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
                                SDValue Ptr, SDValue Mask, SDValue EVL,
                                MachinePointerInfo PtrInfo,
                                MaybeAlign Alignment,
                                MachineMemOperand::Flags MMOFlags,
                                const AAMDNodes &AAInfo, const MDNode *Ranges,
                                bool IsExpanding) { … }

SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
                                SDValue Ptr, SDValue Mask, SDValue EVL,
                                MachineMemOperand *MMO, bool IsExpanding) { … }

SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
                                   EVT VT, SDValue Chain, SDValue Ptr,
                                   SDValue Mask, SDValue EVL,
                                   MachinePointerInfo PtrInfo, EVT MemVT,
                                   MaybeAlign Alignment,
                                   MachineMemOperand::Flags MMOFlags,
                                   const AAMDNodes &AAInfo, bool IsExpanding) { … }

SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
                                   EVT VT, SDValue Chain, SDValue Ptr,
                                   SDValue Mask, SDValue EVL, EVT MemVT,
                                   MachineMemOperand *MMO, bool IsExpanding) { … }

SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
                                       SDValue Base, SDValue Offset,
                                       ISD::MemIndexedMode AM) { … }

SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
                                 SDValue Ptr, SDValue Offset, SDValue Mask,
                                 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
                                 ISD::MemIndexedMode AM, bool IsTruncating,
                                 bool IsCompressing) { … }

SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
                                      SDValue Val, SDValue Ptr, SDValue Mask,
                                      SDValue EVL, MachinePointerInfo PtrInfo,
                                      EVT SVT, Align Alignment,
                                      MachineMemOperand::Flags MMOFlags,
                                      const AAMDNodes &AAInfo,
                                      bool IsCompressing) { … }

SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
                                      SDValue Val, SDValue Ptr, SDValue Mask,
                                      SDValue EVL, EVT SVT,
                                      MachineMemOperand *MMO,
                                      bool IsCompressing) { … }

SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
                                        SDValue Base, SDValue Offset,
                                        ISD::MemIndexedMode AM) { … }

SDValue SelectionDAG::getStridedLoadVP(
    ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
    SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
    SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) { … }

SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
                                       SDValue Ptr, SDValue Stride,
                                       SDValue Mask, SDValue EVL,
                                       MachineMemOperand *MMO,
                                       bool IsExpanding) { … }

SDValue SelectionDAG::getExtStridedLoadVP(
    ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
    SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
    MachineMemOperand *MMO, bool IsExpanding) { … }

SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
                                        SDValue Val, SDValue Ptr,
                                        SDValue Offset, SDValue Stride,
                                        SDValue Mask, SDValue EVL, EVT MemVT,
                                        MachineMemOperand *MMO,
                                        ISD::MemIndexedMode AM,
                                        bool IsTruncating, bool IsCompressing) { … }

SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
                                             SDValue Val, SDValue Ptr,
                                             SDValue Stride, SDValue Mask,
                                             SDValue EVL, EVT SVT,
                                             MachineMemOperand *MMO,
                                             bool IsCompressing) { … }

SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
                                  ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                                  ISD::MemIndexType IndexType) { … }

SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
                                   ArrayRef<SDValue> Ops,
                                   MachineMemOperand *MMO,
                                   ISD::MemIndexType IndexType) { … }

SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
                                    SDValue Base, SDValue Offset, SDValue Mask,
                                    SDValue PassThru, EVT MemVT,
                                    MachineMemOperand *MMO,
                                    ISD::MemIndexedMode AM,
                                    ISD::LoadExtType ExtTy, bool isExpanding) { … }

SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
                                           SDValue Base, SDValue Offset,
                                           ISD::MemIndexedMode AM) { … }

SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
                                     SDValue Val, SDValue Base, SDValue Offset,
                                     SDValue Mask, EVT MemVT,
                                     MachineMemOperand *MMO,
                                     ISD::MemIndexedMode AM, bool IsTruncating,
                                     bool IsCompressing) { … }

SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
                                            SDValue Base, SDValue Offset,
                                            ISD::MemIndexedMode AM) { … }

SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
                                      ArrayRef<SDValue> Ops,
                                      MachineMemOperand *MMO,
                                      ISD::MemIndexType IndexType,
                                      ISD::LoadExtType ExtTy) { … }

SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
                                       ArrayRef<SDValue> Ops,
                                       MachineMemOperand *MMO,
                                       ISD::MemIndexType IndexType,
                                       bool IsTrunc) { … }

SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
                                         const SDLoc &dl, ArrayRef<SDValue> Ops,
                                         MachineMemOperand *MMO,
                                         ISD::MemIndexType IndexType) { … }

SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
                                  EVT MemVT, MachineMemOperand *MMO) { … }

SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
                                  EVT MemVT, MachineMemOperand *MMO) { … }

SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) { … }

SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { … }

SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
                                      SDNodeFlags Flags) { … }

SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
                               SDValue Ptr, SDValue SV, unsigned Align) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              ArrayRef<SDUse> Ops) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              ArrayRef<SDValue> Ops) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                              ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
                              ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              ArrayRef<SDValue> Ops) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
                              SDVTList VTList) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              SDValue N1) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              SDValue N1, SDValue N2) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              SDValue N1, SDValue N2, SDValue N3) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) { … }

SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
                              SDValue N5) { … }

SDVTList SelectionDAG::getVTList(EVT VT) { … }

SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { … }

SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { … }

SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { … }

SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) { … }


/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
/// specified operands.  If the resultant node already exists in the DAG,
/// this does not modify the specified node, instead it returns the node that
/// already exists.  If the resultant node does not exist in the DAG, the
/// input node is returned.  As a degenerate case, if you specify the same
/// input operands as the node already has, the input node is returned.
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { … }

SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { … }

SDNode *SelectionDAG::
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { … }

SDNode *SelectionDAG::
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
                   SDValue Op3, SDValue Op4) { … }

SDNode *SelectionDAG::
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
                   SDValue Op3, SDValue Op4, SDValue Op5) { … }

SDNode *SelectionDAG::
UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) { … }

/// DropOperands - Release the operands and set this node to have
/// zero operands.
void SDNode::DropOperands() { … }

void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
                                  ArrayRef<MachineMemOperand *> NewMemRefs) { … }

/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
/// machine opcode.
///
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT, SDValue Op1) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT, SDValue Op1,
                                   SDValue Op2) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT, SDValue Op1,
                                   SDValue Op2, SDValue Op3) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT, ArrayRef<SDValue> Ops) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT1, EVT VT2) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT1, EVT VT2, EVT VT3,
                                   ArrayRef<SDValue> Ops) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   EVT VT1, EVT VT2,
                                   SDValue Op1, SDValue Op2) { … }

SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
                                   SDVTList VTs,ArrayRef<SDValue> Ops) { … }

/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
/// the line number information on the merged node since it is not possible to
/// preserve the information that operation is associated with multiple lines.
/// This will make the debugger working better at -O0, were there is a higher
/// probability having other instructions associated with that line.
///
/// For IROrder, we keep the smaller of the two
SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { … }

/// MorphNodeTo - This *mutates* the specified node to have the specified
/// return type, opcode, and operands.
///
/// Note that MorphNodeTo returns the resultant node.  If there is already a
/// node of the specified opcode and operands, it returns that node instead of
/// the current one.  Note that the SDLoc need not be the same.
///
/// Using MorphNodeTo is faster than creating a new node and swapping it in
/// with ReplaceAllUsesWith both because it often avoids allocating a new
/// node, and because it doesn't require CSE recalculation for any of
/// the node's users.
///
/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
/// As a consequence it isn't appropriate to use from within the DAG combiner or
/// the legalizer which maintain worklists that would need to be updated when
/// deleting things.
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
                                  SDVTList VTs, ArrayRef<SDValue> Ops) { … }

SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { … }

/// getMachineNode - These are used for target selectors to create a new node
/// with specified return type(s), MachineInstr opcode, and operands.
///
/// Note that getMachineNode returns the resultant node.  If there is already a
/// node of the specified opcode and operands, it returns that node instead of
/// the current one.
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT, SDValue Op1) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT, SDValue Op1, SDValue Op2) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT, SDValue Op1, SDValue Op2,
                                            SDValue Op3) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT, ArrayRef<SDValue> Ops) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT1, EVT VT2, SDValue Op1,
                                            SDValue Op2) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT1, EVT VT2, SDValue Op1,
                                            SDValue Op2, SDValue Op3) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT1, EVT VT2,
                                            ArrayRef<SDValue> Ops) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT1, EVT VT2, EVT VT3,
                                            SDValue Op1, SDValue Op2) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT1, EVT VT2, EVT VT3,
                                            SDValue Op1, SDValue Op2,
                                            SDValue Op3) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            EVT VT1, EVT VT2, EVT VT3,
                                            ArrayRef<SDValue> Ops) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
                                            ArrayRef<EVT> ResultTys,
                                            ArrayRef<SDValue> Ops) { … }

MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
                                            SDVTList VTs,
                                            ArrayRef<SDValue> Ops) { … }

/// getTargetExtractSubreg - A convenience function for creating
/// TargetOpcode::EXTRACT_SUBREG nodes.
SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
                                             SDValue Operand) { … }

/// getTargetInsertSubreg - A convenience function for creating
/// TargetOpcode::INSERT_SUBREG nodes.
SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
                                            SDValue Operand, SDValue Subreg) { … }

/// getNodeIfExists - Get the specified node if it's already available, or
/// else return NULL.
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
                                      ArrayRef<SDValue> Ops) { … }

SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
                                      ArrayRef<SDValue> Ops,
                                      const SDNodeFlags Flags) { … }

/// doesNodeExist - Check if a node exists without modifying its flags.
bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
                                 ArrayRef<SDValue> Ops) { … }

/// getDbgValue - Creates a SDDbgValue node.
///
/// SDNode
SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
                                      SDNode *N, unsigned R, bool IsIndirect,
                                      const DebugLoc &DL, unsigned O) { … }

/// Constant
SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
                                              DIExpression *Expr,
                                              const Value *C,
                                              const DebugLoc &DL, unsigned O) { … }

/// FrameIndex
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
                                                DIExpression *Expr, unsigned FI,
                                                bool IsIndirect,
                                                const DebugLoc &DL,
                                                unsigned O) { … }

/// FrameIndex with dependencies
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
                                                DIExpression *Expr, unsigned FI,
                                                ArrayRef<SDNode *> Dependencies,
                                                bool IsIndirect,
                                                const DebugLoc &DL,
                                                unsigned O) { … }

/// VReg
SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
                                          unsigned VReg, bool IsIndirect,
                                          const DebugLoc &DL, unsigned O) { … }

SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
                                          ArrayRef<SDDbgOperand> Locs,
                                          ArrayRef<SDNode *> Dependencies,
                                          bool IsIndirect, const DebugLoc &DL,
                                          unsigned O, bool IsVariadic) { … }

void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
                                     unsigned OffsetInBits, unsigned SizeInBits,
                                     bool InvalidateDbg) { … }

void SelectionDAG::salvageDebugInfo(SDNode &N) { … }

/// Creates a SDDbgLabel node.
SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
                                      const DebugLoc &DL, unsigned O) { … }

namespace {

/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
/// pointed to by a use iterator is deleted, increment the use iterator
/// so that it doesn't dangle.
///
class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { … };

} // end anonymous namespace

/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
/// This can cause recursive merging of nodes in the DAG.
///
/// This version assumes From has a single result value.
///
void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { … }

/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
/// This can cause recursive merging of nodes in the DAG.
///
/// This version assumes that for each value of From, there is a
/// corresponding value in To in the same position with the same type.
///
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { … }

/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
/// This can cause recursive merging of nodes in the DAG.
///
/// This version can replace From with any result values.  To must match the
/// number and types of values returned by From.
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { … }

/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
/// uses of other values produced by From.getNode() alone.  The Deleted
/// vector is handled the same way as for ReplaceAllUsesWith.
void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ … }

namespace {

/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
/// to record information about a use.
struct UseMemo { … };

/// operator< - Sort Memos by User.
bool operator<(const UseMemo &L, const UseMemo &R) { … }

/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
/// the node already has been taken care of recursively.
class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener { … };

} // end anonymous namespace

/// Return true if a glue output should propagate divergence information.
static bool gluePropagatesDivergence(const SDNode *Node) { … }

bool SelectionDAG::calculateDivergence(SDNode *N) { … }

void SelectionDAG::updateDivergence(SDNode *N) { … }

void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { … }

#ifndef NDEBUG
void SelectionDAG::VerifyDAGDivergence() {
  std::vector<SDNode *> TopoOrder;
  CreateTopologicalOrder(TopoOrder);
  for (auto *N : TopoOrder) {
    assert(calculateDivergence(N) == N->isDivergent() &&
           "Divergence bit inconsistency detected");
  }
}
#endif

/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
/// uses of other values produced by From.getNode() alone.  The same value
/// may appear in both the From and To list.  The Deleted vector is
/// handled the same way as for ReplaceAllUsesWith.
void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
                                              const SDValue *To,
                                              unsigned Num){ … }

/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
/// based on their topological order. It returns the maximum id and a vector
/// of the SDNodes* in assigned order by reference.
unsigned SelectionDAG::AssignTopologicalOrder() { … }

/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
/// value is produced by SD.
void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) { … }

void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { … }

SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
                                                   SDValue NewMemOpChain) { … }

SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
                                                   SDValue NewMemOp) { … }

SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
                                                     Function **OutFunction) { … }

//===----------------------------------------------------------------------===//
//                              SDNode Class
//===----------------------------------------------------------------------===//

bool llvm::isNullConstant(SDValue V) { … }

bool llvm::isNullConstantOrUndef(SDValue V) { … }

bool llvm::isNullFPConstant(SDValue V) { … }

bool llvm::isAllOnesConstant(SDValue V) { … }

bool llvm::isOneConstant(SDValue V) { … }

bool llvm::isMinSignedConstant(SDValue V) { … }

bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
                             unsigned OperandNo) { … }

SDValue llvm::peekThroughBitcasts(SDValue V) { … }

SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { … }

SDValue llvm::peekThroughExtractSubvectors(SDValue V) { … }

SDValue llvm::peekThroughTruncates(SDValue V) { … }

bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { … }

ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
                                          bool AllowTruncation) { … }

ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                          bool AllowUndefs,
                                          bool AllowTruncation) { … }

ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { … }

ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
                                              const APInt &DemandedElts,
                                              bool AllowUndefs) { … }

bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { … }

bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) { … }

bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) { … }

HandleSDNode::~HandleSDNode() { … }

MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                     SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
    : … { … }

/// Profile - Gather unique data for the node.
///
void SDNode::Profile(FoldingSetNodeID &ID) const { … }

namespace {

  struct EVTArray { … };

} // end anonymous namespace

/// getValueTypeList - Return a pointer to the specified value type.
///
const EVT *SDNode::getValueTypeList(EVT VT) { … }

/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
/// indicated value.  This method ignores uses of other values defined by this
/// operation.
bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { … }

/// hasAnyUseOfValue - Return true if there are any use of the indicated
/// value. This method ignores uses of other values defined by this operation.
bool SDNode::hasAnyUseOfValue(unsigned Value) const { … }

/// isOnlyUserOf - Return true if this node is the only use of N.
bool SDNode::isOnlyUserOf(const SDNode *N) const { … }

/// Return true if the only users of N are contained in Nodes.
bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { … }

/// isOperand - Return true if this node is an operand of N.
bool SDValue::isOperandOf(const SDNode *N) const { … }

bool SDNode::isOperandOf(const SDNode *N) const { … }

/// reachesChainWithoutSideEffects - Return true if this operand (which must
/// be a chain) reaches the specified operand without crossing any
/// side-effecting instructions on any chain path.  In practice, this looks
/// through token factors and non-volatile loads.  In order to remain efficient,
/// this only looks a couple of nodes in, it does not do an exhaustive search.
///
/// Note that we only need to examine chains when we're searching for
/// side-effects; SelectionDAG requires that all side-effects are represented
/// by chains, even if another operand would force a specific ordering. This
/// constraint is necessary to allow transformations like splitting loads.
bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
                                             unsigned Depth) const { … }

bool SDNode::hasPredecessor(const SDNode *N) const { … }

void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { … }

SDValue
SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
                                  ArrayRef<ISD::NodeType> CandidateBinOps,
                                  bool AllowPartials) { … }

SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { … }

std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
    SDNode *N, unsigned ResNE) { … }

bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
                                                  LoadSDNode *Base,
                                                  unsigned Bytes,
                                                  int Dist) const { … }

/// InferPtrAlignment - Infer alignment of a load / store address. Return
/// std::nullopt if it cannot be inferred.
MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const { … }

/// Split the scalar node with EXTRACT_ELEMENT using the provided
/// VTs and return the low/high part.
std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
                                                      const SDLoc &DL,
                                                      const EVT &LoVT,
                                                      const EVT &HiVT) { … }

/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
/// which is split (or expanded) into two not necessarily identical pieces.
std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { … }

/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
/// type, dependent on an enveloping VT that has been split into two identical
/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
std::pair<EVT, EVT>
SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
                                       bool *HiIsEmpty) const { … }

/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
/// low/high part.
std::pair<SDValue, SDValue>
SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
                          const EVT &HiVT) { … }

std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
                                                   const SDLoc &DL) { … }

/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { … }

void SelectionDAG::ExtractVectorElements(SDValue Op,
                                         SmallVectorImpl<SDValue> &Args,
                                         unsigned Start, unsigned Count,
                                         EVT EltVT) { … }

// getAddressSpace - Return the address space this GlobalAddress belongs to.
unsigned GlobalAddressSDNode::getAddressSpace() const { … }

Type *ConstantPoolSDNode::getType() const { … }

bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                                        unsigned &SplatBitSize,
                                        bool &HasAnyUndefs,
                                        unsigned MinSplatBits,
                                        bool IsBigEndian) const { … }

SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
                                         BitVector *UndefElements) const { … }

SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { … }

bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
                                            SmallVectorImpl<SDValue> &Sequence,
                                            BitVector *UndefElements) const { … }

bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                                            BitVector *UndefElements) const { … }

ConstantSDNode *
BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
                                        BitVector *UndefElements) const { … }

ConstantSDNode *
BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { … }

ConstantFPSDNode *
BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
                                          BitVector *UndefElements) const { … }

ConstantFPSDNode *
BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { … }

int32_t
BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                                   uint32_t BitWidth) const { … }

bool BuildVectorSDNode::getConstantRawBits(
    bool IsLittleEndian, unsigned DstEltSizeInBits,
    SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const { … }

void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
                                      unsigned DstEltSizeInBits,
                                      SmallVectorImpl<APInt> &DstBitElements,
                                      ArrayRef<APInt> SrcBitElements,
                                      BitVector &DstUndefElements,
                                      const BitVector &SrcUndefElements) { … }

bool BuildVectorSDNode::isConstant() const { … }

std::optional<std::pair<APInt, APInt>>
BuildVectorSDNode::isConstantSequence() const { … }

bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { … }

// Returns the SDNode if it is a constant integer BuildVector
// or constant integer.
SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { … }

// Returns the SDNode if it is a constant float BuildVector
// or constant float.
SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { … }

std::optional<bool> SelectionDAG::isBoolConstant(SDValue N,
                                                 bool AllowTruncation) const { … }

void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { … }

SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
                                     SmallVectorImpl<SDValue> &Vals) { … }

SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
                                        EVT VT, SDNodeFlags Flags) { … }

/// Helper used to make a call to a library function that has one argument of
/// pointer type.
///
/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
/// used to get or set floating-point state. They have one argument of pointer
/// type, which points to the memory region containing bits of the
/// floating-point state. The value returned by such function is ignored in the
/// created call.
///
/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
/// \param Ptr Pointer used to save/load state.
/// \param InChain Ingoing token chain.
/// \returns Outgoing chain token.
SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
                                            SDValue InChain,
                                            const SDLoc &DLoc) { … }

void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) { … }

#ifndef NDEBUG
static void checkForCyclesHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode*> &Visited,
                                 SmallPtrSetImpl<const SDNode*> &Checked,
                                 const llvm::SelectionDAG *DAG) {
  // If this node has already been checked, don't check it again.
  if (Checked.count(N))
    return;

  // If a node has already been visited on this depth-first walk, reject it as
  // a cycle.
  if (!Visited.insert(N).second) {
    errs() << "Detected cycle in SelectionDAG\n";
    dbgs() << "Offending node:\n";
    N->dumprFull(DAG); dbgs() << "\n";
    abort();
  }

  for (const SDValue &Op : N->op_values())
    checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);

  Checked.insert(N);
  Visited.erase(N);
}
#endif

void llvm::checkForCycles(const llvm::SDNode *N,
                          const llvm::SelectionDAG *DAG,
                          bool force) { … }

void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { … }