//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file performs vector type splitting and scalarization for LegalizeTypes. // Scalarization is the act of changing a computation in an illegal one-element // vector type to be a computation in its scalar element type. For example, // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed // as a base case when scalarizing vector arithmetic like <4 x f32>, which // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32 // types. // Splitting is the act of changing a computation in an invalid vector type to // be a computation in two vectors of half the size. For example, implementing // <128 x f32> operations in terms of two <64 x f32> operations. // //===----------------------------------------------------------------------===// #include "LegalizeTypes.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/IR/DataLayout.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TypeSize.h" #include "llvm/Support/raw_ostream.h" #include <numeric> usingnamespacellvm; #define DEBUG_TYPE … //===----------------------------------------------------------------------===// // Result Vector Scalarization: <1 x ty> -> ty. //===----------------------------------------------------------------------===// void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_CMP(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_FFREXP(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_ExpOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_ADDRSPACECAST(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecRes_IS_FPCLASS(SDNode *N) { … } //===----------------------------------------------------------------------===// // Operand Vector Scalarization <1 x ty> -> ty. //===----------------------------------------------------------------------===// bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { … } /// If the value to convert is a vector that needs to be scalarized, it must be /// <1 x ty>. Convert the element instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) { … } // Need to legalize vector operands of fake uses. Must be <1 x ty>. SDValue DAGTypeLegalizer::ScalarizeVecOp_FAKE_USE(SDNode *N) { … } /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. /// Do the operation on the element instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) { … } /// If the input is a vector that needs to be scalarized, it must be <1 x ty>. /// Do the strict FP operation on the element instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) { … } /// The vectors to concatenate have length one - use a BUILD_VECTOR instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) { … } /// The inserted subvector is to be scalarized - use insert vector element /// instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo) { … } /// If the input is a vector that needs to be scalarized, it must be <1 x ty>, /// so just return the element, ignoring the index. SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { … } /// If the input condition is a vector that needs to be scalarized, it must be /// <1 x i1>, so just convert to a normal ISD::SELECT /// (still with vector output type since that was acceptable if we got here). SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) { … } /// If the operand is a vector that needs to be scalarized then the /// result must be v1i1, so just convert to a scalar SETCC and wrap /// with a scalar_to_vector since the res type is legal if we got here SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) { … } /// If the value to store is a vector that needs to be scalarized, it must be /// <1 x ty>. Just store the element. SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ … } /// If the value to round is a vector that needs to be scalarized, it must be /// <1 x ty>. Convert the element instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo) { … } /// If the value to extend is a vector that needs to be scalarized, it must be /// <1 x ty>. Convert the element instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) { … } /// If the value to extend is a vector that needs to be scalarized, it must be /// <1 x ty>. Convert the element instead. SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) { … } SDValue DAGTypeLegalizer::ScalarizeVecOp_CMP(SDNode *N) { … } //===----------------------------------------------------------------------===// // Result Vector Splitting //===----------------------------------------------------------------------===// /// This method is called when the specified result of the specified node is /// found to need vector splitting. At this point, the node may also have /// invalid operands or may have other results that need legalization, we just /// know that (at least) one result needs vector splitting. void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { … } void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT, MachinePointerInfo &MPI, SDValue &Ptr, uint64_t *ScaledOffset) { … } std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask) { … } std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(SDValue Mask, const SDLoc &DL) { … } void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi) { … } // Handle splitting an FP where the second operand does not match the first // type. The second operand may be a scalar, or a vector that has exactly as // many elements as the first void DAGTypeLegalizer::SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { … } void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VP_SPLAT(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo, SDValue &Hi, bool SplitSETCC) { … } void DAGTypeLegalizer::SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_FFREXP(SDNode *N, unsigned ResNo, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi) { … } void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) { … } void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) { … } //===----------------------------------------------------------------------===// // Operand Vector Splitting //===----------------------------------------------------------------------===// /// This method is called when the specified operand of the specified node is /// found to need vector splitting. At this point, all of the result types of /// the node are known to be legal, but other operands of the node may need /// legalization as well as the specified one. bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { … } // Split a FAKE_USE use of a vector into FAKE_USEs of hi and lo part. SDValue DAGTypeLegalizer::SplitVecOp_FAKE_USE(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_Gather(MemSDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { … } // Split a vector type in an FP binary operation where the second operand has a // different type from the first. // // The result (and the first input) has a legal vector type, but the second // input needs splitting. SDValue DAGTypeLegalizer::SplitVecOp_FPOpDifferentTypes(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_CMP(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_VP_CttzElements(SDNode *N) { … } SDValue DAGTypeLegalizer::SplitVecOp_VECTOR_HISTOGRAM(SDNode *N) { … } //===----------------------------------------------------------------------===// // Result Vector Widening //===----------------------------------------------------------------------===// void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_CMP(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) { … } // Given a vector of operations that have been broken up to widen, see // if we can collect them together into the next widest legal VT. This // implementation is trap-safe. static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, SmallVectorImpl<SDValue> &ConcatOps, unsigned ConcatEnd, EVT VT, EVT MaxVT, EVT WidenVT) { … } SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_XROUND(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) { … } /// Result and first source operand are different scalar types, but must have /// the same number of elements. There is an additional control argument which /// should be passed through unchanged. SDValue DAGTypeLegalizer::WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_ExpOp(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { … } SDValue DAGTypeLegalizer::WidenVecRes_ADDRSPACECAST(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_INSERT_SUBVECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_AssertZext(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_VP_LOAD(VPLoadSDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_COMPRESS(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) { … } // Return true is this is a SETCC node or a strict version of it. static inline bool isSETCCOp(unsigned Opcode) { … } // Return true if this is a node that could have two SETCCs as operands. static inline bool isLogicalMaskOp(unsigned Opcode) { … } // If N is a SETCC or a strict variant of it, return the type // of the compare operands. static inline EVT getSETCCOperandType(SDValue N) { … } // This is used just for the assert in convertMask(). Check that this either // a SETCC or a previously handled SETCC by convertMask(). #ifndef NDEBUG static inline bool isSETCCorConvertedSETCC(SDValue N) { if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) N = N.getOperand(0); else if (N.getOpcode() == ISD::CONCAT_VECTORS) { for (unsigned i = 1; i < N->getNumOperands(); ++i) if (!N->getOperand(i)->isUndef()) return false; N = N.getOperand(0); } if (N.getOpcode() == ISD::TRUNCATE) N = N.getOperand(0); else if (N.getOpcode() == ISD::SIGN_EXTEND) N = N.getOperand(0); if (isLogicalMaskOp(N.getOpcode())) return isSETCCorConvertedSETCC(N.getOperand(0)) && isSETCCorConvertedSETCC(N.getOperand(1)); return (isSETCCOp(N.getOpcode()) || ISD::isBuildVectorOfConstantSDNodes(N.getNode())); } #endif // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT // to ToMaskVT if needed with vector extension or truncation. SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT) { … } // This method tries to handle some special cases for the vselect mask // and if needed adjusting the mask vector type to match that of the VSELECT. // Without it, many cases end up with scalarization of the SETCC, with many // unnecessary instructions. SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_Select(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) { … } //===----------------------------------------------------------------------===// // Widen Vector Operand //===----------------------------------------------------------------------===// bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_CMP(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_UnrollVectorOp(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_IS_FPCLASS(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { … } // Vectors with sizes that are not powers of 2 need to be widened to the // next largest power of 2. For example, we may get a vector of 3 32-bit // integers or of 6 16-bit integers, both of which have to be widened to a // 128-bit vector. SDValue DAGTypeLegalizer::WidenVecOp_FAKE_USE(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_VP_SPLAT(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_VP_STRIDED_STORE(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_VP_SCATTER(SDNode *N, unsigned OpNo) { … } SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) { … } static unsigned getExtendForIntVecReduction(unsigned Opc) { … } SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_VP_REDUCE(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { … } SDValue DAGTypeLegalizer::WidenVecOp_VP_CttzElements(SDNode *N) { … } //===----------------------------------------------------------------------===// // Vector Widening Utilities //===----------------------------------------------------------------------===// // Utility function to find the type to chop up a widen vector for load/store // TLI: Target lowering used to determine legal types. // Width: Width left need to load/store. // WidenVT: The widen vector type to load to/store from // Align: If 0, don't allow use of a wider type // WidenEx: If Align is not 0, the amount additional we can load/store from. static std::optional<EVT> findMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align = 0, unsigned WidenEx = 0) { … } // Builds a vector type from scalar loads // VecTy: Resulting Vector type // LDOps: Load operators to build a vector type // [Start,End) the list of loads to use. static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy, SmallVectorImpl<SDValue> &LdOps, unsigned Start, unsigned End) { … } SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, LoadSDNode *LD) { … } SDValue DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, LoadSDNode *LD, ISD::LoadExtType ExtType) { … } bool DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST) { … } /// Modifies a vector input (widen or narrows) to a vector of NVT. The /// input vector must have the same element type as NVT. /// FillWithZeroes specifies that the vector should be widened with zeroes. SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes) { … }
Codebase Browser
llvm