//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This implements the TargetLowering class. // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/TargetLowering.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/CodeGenCommonISel.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/LLVMContext.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCExpr.h" #include "llvm/Support/DivisionByConstantInfo.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" #include "llvm/Target/TargetMachine.h" #include <cctype> usingnamespacellvm; /// NOTE: The TargetMachine owns TLOF. TargetLowering::TargetLowering(const TargetMachine &tm) : … { … } const char *TargetLowering::getTargetNodeName(unsigned Opcode) const { … } bool TargetLowering::isPositionIndependent() const { … } /// Check whether a given call node is in tail position within its function. If /// so, it sets Chain to the input chain of the tail call. bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const { … } bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl<CCValAssign> &ArgLocs, const SmallVectorImpl<SDValue> &OutVals) const { … } /// Set CallLoweringInfo attribute flags based on a call instruction /// and called function attributes. void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, unsigned ArgIdx) { … } /// Generate a libcall taking the given operands as arguments and returning a /// result of type RetVT. std::pair<SDValue, SDValue> TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue InChain) const { … } bool TargetLowering::findOptimalMemOpLowering( std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const { … } /// Soften the operands of a comparison. This code is shared among BR_CC, /// SELECT_CC, and SETCC handlers. void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &dl, const SDValue OldLHS, const SDValue OldRHS) const { … } void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &dl, const SDValue OldLHS, const SDValue OldRHS, SDValue &Chain, bool IsSignaling) const { … } /// Return the entry encoding for a jump table in the current function. The /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum. unsigned TargetLowering::getJumpTableEncoding() const { … } SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const { … } /// This returns the relocation base for the given PIC jumptable, the same as /// getPICJumpTableRelocBase, but as an MCExpr. const MCExpr * TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,MCContext &Ctx) const{ … } SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const { … } bool TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { … } //===----------------------------------------------------------------------===// // Optimization Methods //===----------------------------------------------------------------------===// /// If the specified instruction has a constant integer operand and there are /// bits set in that constant that are not demanded, then clear those bits and /// return true. bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const { … } bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, TargetLoweringOpt &TLO) const { … } /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free. /// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast, /// but it could be generalized for targets with other types of implicit /// widening casts. bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &DemandedBits, TargetLoweringOpt &TLO) const { … } bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, DAGCombinerInfo &DCI) const { … } bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, DAGCombinerInfo &DCI) const { … } bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth, bool AssumeSingleUse) const { … } // TODO: Under what circumstances can we create nodes? Constant folding? SDValue TargetLowering::SimplifyMultipleUseDemandedBits( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const { … } SDValue TargetLowering::SimplifyMultipleUseDemandedBits( SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG, unsigned Depth) const { … } SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts( SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const { … } // Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1). // or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1). static SDValue combineShiftToAVG(SDValue Op, TargetLowering::TargetLoweringOpt &TLO, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth) { … } /// Look at Op. At this point, we know that only the OriginalDemandedBits of the /// result of Op are ever used downstream. If we can use this information to /// simplify Op, create a new simplified DAG node and return true, returning the /// original and new nodes in Old and New. Otherwise, analyze the expression and /// return a mask of Known bits for the expression (used to simplify the /// caller). The Known bits may only be accurate for those bits in the /// OriginalDemandedBits and OriginalDemandedElts. bool TargetLowering::SimplifyDemandedBits( SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth, bool AssumeSingleUse) const { … } bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts, DAGCombinerInfo &DCI) const { … } /// Given a vector binary operation and known undefined elements for each input /// operand, compute whether each element of the output is undefined. static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG, const APInt &UndefOp0, const APInt &UndefOp1) { … } bool TargetLowering::SimplifyDemandedVectorElts( SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth, bool AssumeSingleUse) const { … } /// Determine which of the bits specified in Mask are known to be either zero or /// one and return them in the Known. void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const { … } void TargetLowering::computeKnownBitsForTargetInstr( GISelKnownBits &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth) const { … } void TargetLowering::computeKnownBitsForFrameIndex( const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const { … } Align TargetLowering::computeKnownAlignForTargetInstr( GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth) const { … } /// This method can be implemented by targets that want to expose additional /// information about sign bits to the DAG Combiner. unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &, const SelectionDAG &, unsigned Depth) const { … } unsigned TargetLowering::computeNumSignBitsForTargetInstr( GISelKnownBits &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth) const { … } bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode( SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth) const { … } bool TargetLowering::SimplifyDemandedBitsForTargetNode( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const { … } SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const { … } SDValue TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef<int> Mask, SelectionDAG &DAG) const { … } const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const { … } bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const { … } bool TargetLowering::canCreateUndefOrPoisonForTargetNode( SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const { … } bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN, unsigned Depth) const { … } bool TargetLowering::isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth) const { … } // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must // work with truncating build vectors and vectors with elements of less than // 8 bits. bool TargetLowering::isConstTrueVal(SDValue N) const { … } bool TargetLowering::isConstFalseVal(SDValue N) const { … } bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const { … } /// This helper function of SimplifySetCC tries to optimize the comparison when /// either operand of the SetCC node is a bitwise-and instruction. SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &DL, DAGCombinerInfo &DCI) const { … } /// There are multiple IR patterns that could be checking whether certain /// truncation of a signed number would be lossy or not. The pattern which is /// best at IR level, may not lower optimally. Thus, we want to unfold it. /// We are looking for the following pattern: (KeptBits is a constant) /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits) /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false. /// KeptBits also can't be 1, that would have been folded to %x dstcond 0 /// We will unfold it into the natural trunc+sext pattern: /// ((%x << C) a>> C) dstcond %x /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x) SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck( EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL) const { … } // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift( EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL) const { … } /// Try to fold an equality comparison with a {add/sub/xor} binary operation as /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to /// handle the commuted versions of these patterns. SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &DL, DAGCombinerInfo &DCI) const { … } static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, SDValue N0, const APInt &C1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG) { … } static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG) { … } static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG) { … } /// Try to simplify a setcc built with the specified operands and cc. If it is /// unable to simplify it, return a null SDValue. SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const { … } /// Returns true (and the GlobalValue and the offset) if the node is a /// GlobalAddress + offset. bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA, int64_t &Offset) const { … } SDValue TargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { … } //===----------------------------------------------------------------------===// // Inline Assembler Implementation Methods //===----------------------------------------------------------------------===// TargetLowering::ConstraintType TargetLowering::getConstraintType(StringRef Constraint) const { … } /// Try to replace an X constraint, which matches anything, with another that /// has more specific requirements based on the type of the corresponding /// operand. const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const { … } SDValue TargetLowering::LowerAsmOutputForConstraint( SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const { … } /// Lower the specified operand into the Ops vector. /// If it is invalid, don't add anything to Ops. void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops, SelectionDAG &DAG) const { … } void TargetLowering::CollectTargetIntrinsicOperands( const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const { … } std::pair<unsigned, const TargetRegisterClass *> TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI, StringRef Constraint, MVT VT) const { … } //===----------------------------------------------------------------------===// // Constraint Selection. /// Return true of this is an input operand that is a matching constraint like /// "4". bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const { … } /// If this is an input matching constraint, this method returns the output /// operand it matches. unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const { … } /// Split up the constraint string from the inline assembly value into the /// specific constraints and their prefixes, and also tie in the associated /// operand values. /// If this returns an empty vector, and if the constraint string itself /// isn't empty, there was an error parsing. TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const { … } /// Return a number indicating our preference for chosing a type of constraint /// over another, for the purpose of sorting them. Immediates are almost always /// preferrable (when they can be emitted). A higher return value means a /// stronger preference for one constraint type relative to another. /// FIXME: We should prefer registers over memory but doing so may lead to /// unrecoverable register exhaustion later. /// https://github.com/llvm/llvm-project/issues/20571 static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) { … } /// Examine constraint type and operand type and determine a weight value. /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight TargetLowering::getMultipleConstraintMatchWeight( AsmOperandInfo &info, int maIndex) const { … } /// Examine constraint type and operand type and determine a weight value. /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight TargetLowering::getSingleConstraintMatchWeight( AsmOperandInfo &info, const char *constraint) const { … } /// If there are multiple different constraints that we could pick for this /// operand (e.g. "imr") try to pick the 'best' one. /// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall /// into seven classes: /// Register -> one specific register /// RegisterClass -> a group of regs /// Memory -> memory /// Address -> a symbolic memory reference /// Immediate -> immediate values /// Other -> magic values (such as "Flag Output Operands") /// Unknown -> something we don't recognize yet and can't handle /// Ideally, we would pick the most specific constraint possible: if we have /// something that fits into a register, we would pick it. The problem here /// is that if we have something that could either be in a register or in /// memory that use of the register could cause selection of *other* /// operands to fail: they might only succeed if we pick memory. Because of /// this the heuristic we use is: /// /// 1) If there is an 'other' constraint, and if the operand is valid for /// that constraint, use it. This makes us take advantage of 'i' /// constraints when available. /// 2) Otherwise, pick the most general constraint present. This prefers /// 'm' over 'r', for example. /// TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences( TargetLowering::AsmOperandInfo &OpInfo) const { … } /// If we have an immediate, see if we can lower it. Return true if we can, /// false otherwise. static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, SDValue Op, SelectionDAG *DAG, const TargetLowering &TLI) { … } /// Determines the constraint code and constraint type to use for the specific /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType. void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG) const { … } /// Given an exact SDIV by a constant, create a multiplication /// with the multiplicative inverse of the constant. /// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242 static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl<SDNode *> &Created) { … } /// Given an exact UDIV by a constant, create a multiplication /// with the multiplicative inverse of the constant. /// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242 static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl<SDNode *> &Created) { … } SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl<SDNode *> &Created) const { … } SDValue TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl<SDNode *> &Created) const { … } /// Build sdiv by power-of-2 with conditional move instructions /// Ref: "Hacker's Delight" by Henry Warren 10-1 /// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into: /// bgez x, label /// add x, x, 2**k-1 /// label: /// sra res, x, k /// neg res, res (when the divisor is negative) SDValue TargetLowering::buildSDIVPow2WithCMov( SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl<SDNode *> &Created) const { … } /// Given an ISD::SDIV node expressing a divide by constant, /// return a DAG expression to select that will generate the same value by /// multiplying by a magic number. /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide". SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, bool IsAfterLegalTypes, SmallVectorImpl<SDNode *> &Created) const { … } /// Given an ISD::UDIV node expressing a divide by constant, /// return a DAG expression to select that will generate the same value by /// multiplying by a magic number. /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide". SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, bool IsAfterLegalTypes, SmallVectorImpl<SDNode *> &Created) const { … } /// If all values in Values that *don't* match the predicate are same 'splat' /// value, then replace all values with that splat value. /// Else, if AlternativeReplacement was provided, then replace all values that /// do match predicate with AlternativeReplacement value. static void turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values, std::function<bool(SDValue)> Predicate, SDValue AlternativeReplacement = SDValue()) { … } /// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE /// where the divisor is constant and the comparison target is zero, /// return a DAG expression that will generate the same comparison result /// using only multiplications, additions and shifts/rotations. /// Ref: "Hacker's Delight" 10-17. SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL) const { … } SDValue TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL, SmallVectorImpl<SDNode *> &Created) const { … } /// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE /// where the divisor is constant and the comparison target is zero, /// return a DAG expression that will generate the same comparison result /// using only multiplications, additions and shifts/rotations. /// Ref: "Hacker's Delight" 10-17. SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL) const { … } SDValue TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL, SmallVectorImpl<SDNode *> &Created) const { … } bool TargetLowering:: verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const { … } SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG, const DenormalMode &Mode) const { … } SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth) const { … } //===----------------------------------------------------------------------===// // Legalization Utilities //===----------------------------------------------------------------------===// bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl<SDValue> &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL, SDValue LH, SDValue RL, SDValue RH) const { … } bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL, SDValue LH, SDValue RL, SDValue RH) const { … } // Optimize unsigned division or remainder by constants for types twice as large // as a legal VT. // // If (1 << (BitWidth / 2)) % Constant == 1, then the remainder // can be computed // as: // Sum += __builtin_uadd_overflow(Lo, High, &Sum); // Remainder = Sum % Constant // This is based on "Remainder by Summing Digits" from Hacker's Delight. // // For division, we can compute the remainder using the algorithm described // above, subtract it from the dividend to get an exact multiple of Constant. // Then multiply that exact multiply by the multiplicative inverse modulo // (1 << (BitWidth / 2)) to get the quotient. // If Constant is even, we can shift right the dividend and the divisor by the // number of trailing zeros in Constant before applying the remainder algorithm. // If we're after the quotient, we can subtract this value from the shifted // dividend and multiply by the multiplicative inverse of the shifted divisor. // If we want the remainder, we shift the value left by the number of trailing // zeros and add the bits that were shifted out of the dividend. bool TargetLowering::expandDIVREMByConstant(SDNode *N, SmallVectorImpl<SDValue> &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const { … } // Check that (every element of) Z is undef or not an exact multiple of BW. static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) { … } static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG) { … } SDValue TargetLowering::expandFunnelShift(SDNode *Node, SelectionDAG &DAG) const { … } // TODO: Merge with expandFunnelShift. SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps, SelectionDAG &DAG) const { … } void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const { … } bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const { … } bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const { … } bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const { … } SDValue TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node, SelectionDAG &DAG) const { … } /// Returns a true value if if this FPClassTest can be performed with an ordered /// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns /// std::nullopt if it cannot be performed as a compare with 0. static std::optional<bool> isFCmpEqualZero(FPClassTest Test, const fltSemantics &Semantics, const MachineFunction &MF) { … } SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op, const FPClassTest OrigTestMask, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const { … } // Only expand vector types if we have the appropriate vector bit operations. static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) { … } SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVPCTPOP(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned BitWidth) const { … } SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative) const { … } SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVPBSWAP(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const { … } std::pair<SDValue, SDValue> TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const { … } SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const { … } std::pair<SDValue, SDValue> TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const { … } SDValue TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const { … } static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &dl, ElementCount SubEC) { … } SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const { … } SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index) const { … } //===----------------------------------------------------------------------===// // Implementation of Emulated TLS Model //===----------------------------------------------------------------------===// SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const { … } SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const { … } void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const { … } void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, const SDValue LHS, const SDValue RHS, SDValue &Lo, SDValue &Hi) const { … } SDValue TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const { … } void TargetLowering::expandUADDSUBO( SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const { … } void TargetLowering::expandSADDSUBO( SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const { … } bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const { … } bool TargetLowering::expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &dl, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const { … } SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const { … } bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, SDValue Mask, SDValue EVL, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling) const { … }
Codebase Browser
llvm