//===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // /// \file This file implements the LegalizerHelper class to legalize /// individual instructions and the LegalizeMachineIR wrapper pass for the /// primary legalization. // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RuntimeLibcallUtil.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/Instructions.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include <numeric> #include <optional> #define DEBUG_TYPE … usingnamespacellvm; usingnamespaceLegalizeActions; usingnamespaceMIPatternMatch; /// Try to break down \p OrigTy into \p NarrowTy sized pieces. /// /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy, /// with any leftover piece as type \p LeftoverTy /// /// Returns -1 in the first element of the pair if the breakdown is not /// satisfiable. static std::pair<int, int> getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { … } static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) { … } LegalizerHelper::LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &Builder) : … { … } LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, GISelChangeObserver &Observer, MachineIRBuilder &B, GISelKnownBits *KB) : … { … } LegalizerHelper::LegalizeResult LegalizerHelper::legalizeInstrStep(MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } void LegalizerHelper::insertParts(Register DstReg, LLT ResultTy, LLT PartTy, ArrayRef<Register> PartRegs, LLT LeftoverTy, ArrayRef<Register> LeftoverRegs) { … } void LegalizerHelper::appendVectorElts(SmallVectorImpl<Register> &Elts, Register Reg) { … } /// Merge \p PartRegs with different types into \p DstReg. void LegalizerHelper::mergeMixedSubvectors(Register DstReg, ArrayRef<Register> PartRegs) { … } /// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs. static void getUnmergeResults(SmallVectorImpl<Register> &Regs, const MachineInstr &MI) { … } void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, Register SrcReg) { … } LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, LLT NarrowTy, Register SrcReg) { … } LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, SmallVectorImpl<Register> &VRegs, unsigned PadStrategy) { … } void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, ArrayRef<Register> RemergeRegs) { … } static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { … } /// True if an instruction is in tail position in its caller. Intended for /// legalizing libcalls as tail calls when possible. static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result, MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI) { … } LegalizerHelper::LegalizeResult llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, const CallLowering::ArgInfo &Result, ArrayRef<CallLowering::ArgInfo> Args, const CallingConv::ID CC, LostDebugLocObserver &LocObserver, MachineInstr *MI) { … } LegalizerHelper::LegalizeResult llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result, ArrayRef<CallLowering::ArgInfo> Args, LostDebugLocObserver &LocObserver, MachineInstr *MI) { … } // Useful for libcalls where all operands have the same type. static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType, LostDebugLocObserver &LocObserver) { … } LegalizerHelper::LegalizeResult llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) { … } static LegalizerHelper::LegalizeResult createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI) { … } static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType) { … } static LegalizerHelper::LegalizeResult conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, Type *FromType, LostDebugLocObserver &LocObserver) { … } static RTLIB::Libcall getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) { … } // Some library functions that read FP state (fegetmode, fegetenv) write the // state into a region in memory. IR intrinsics that do the same operations // (get_fpmode, get_fpenv) return the state as integer value. To implement these // intrinsics via the library functions, we need to use temporary variable, // for example: // // %0:_(s32) = G_GET_FPMODE // // is transformed to: // // %1:_(p0) = G_FRAME_INDEX %stack.0 // BL &fegetmode // %0:_(s32) = G_LOAD % 1 // LegalizerHelper::LegalizeResult LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } // Similar to `createGetStateLibcall` the function calls a library function // using transient space in stack. In this case the library function reads // content of memory region. LegalizerHelper::LegalizeResult LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } /// Returns the corresponding libcall for the given Pred and /// the ICMP predicate that should be generated to compare with #0 /// after the libcall. static std::pair<RTLIB::Libcall, CmpInst::Predicate> getFCMPLibcallDesc(const CmpInst::Predicate Pred) { … } LegalizerHelper::LegalizeResult LegalizerHelper::createFCMPLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } // The function is used to legalize operations that set default environment // state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that. // On most targets supported in glibc FE_DFL_MODE is defined as // `((const femode_t *) -1)`. Such assumption is used here. If for some target // it is not true, the target must provide custom lowering. LegalizerHelper::LegalizeResult LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } LegalizerHelper::LegalizeResult LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } Register LegalizerHelper::coerceToScalar(Register Val) { … } void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned ExtOpcode) { … } void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx) { … } void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned TruncOpcode) { … } void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, unsigned ExtOpcode) { … } void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx) { … } void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx) { … } void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { … } void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { … } static void getUnmergePieces(SmallVectorImpl<Register> &Pieces, MachineIRBuilder &B, Register Src, LLT Ty) { … } static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerConstant(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFConstant(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerBitcast(MachineInstr &MI) { … } /// Figure out the bit offset into a register when coercing a vector index for /// the wide element type. This is only for the case when promoting vector to /// one with larger elements. // /// /// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) /// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize) static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize) { … } /// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this /// is casting to a vector with a smaller element size, perform multiple element /// extracts and merge the results. If this is coercing to a vector with larger /// elements, index the bitcasted vector and extract the target element with bit /// operations. This is intended to force the indexing in the native register /// size for architectures that can dynamically index the register file. LegalizerHelper::LegalizeResult LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { … } /// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p /// TargetReg, while preserving other bits in \p TargetReg. /// /// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset) static Register buildBitFieldInsert(MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits) { … } /// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this /// is increasing the element size, perform the indexing in the target element /// type, and use bit operations to insert at the element position. This is /// intended for architectures that can dynamically index the register file and /// want to force indexing in the native register size. LegalizerHelper::LegalizeResult LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { … } // This attempts to handle G_CONCAT_VECTORS with illegal operands, particularly // those that have smaller than legal operands. // // <16 x s8> = G_CONCAT_VECTORS <4 x s8>, <4 x s8>, <4 x s8>, <4 x s8> // // ===> // // s32 = G_BITCAST <4 x s8> // s32 = G_BITCAST <4 x s8> // s32 = G_BITCAST <4 x s8> // s32 = G_BITCAST <4 x s8> // <4 x s32> = G_BUILD_VECTOR s32, s32, s32, s32 // <16 x s8> = G_BITCAST <4 x s32> LegalizerHelper::LegalizeResult LegalizerHelper::bitcastConcatVector(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { … } /// This attempts to bitcast G_EXTRACT_SUBVECTOR to CastTy. /// /// <vscale x 8 x i1> = G_EXTRACT_SUBVECTOR <vscale x 16 x i1>, N /// /// ===> /// /// <vscale x 2 x i1> = G_BITCAST <vscale x 16 x i1> /// <vscale x 1 x i8> = G_EXTRACT_SUBVECTOR <vscale x 2 x i1>, N / 8 /// <vscale x 8 x i1> = G_BITCAST <vscale x 1 x i8> LegalizerHelper::LegalizeResult LegalizerHelper::bitcastExtractSubvector(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) { … } // Legalize an instruction by changing the opcode in place. void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { … } Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty, Align MinAlign) const { … } MachineInstrBuilder LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo) { … } static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg, LLT VecTy) { … } Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index) { … } #ifndef NDEBUG /// Check that all vector operands have same number of elements. Other operands /// should be listed in NonVecOp. static bool hasSameNumEltsOnAllVectorOperands( GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list<unsigned> NonVecOpIndices) { if (MI.getNumMemOperands() != 0) return false; LLT VecTy = MRI.getType(MI.getReg(0)); if (!VecTy.isVector()) return false; unsigned NumElts = VecTy.getNumElements(); for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) { MachineOperand &Op = MI.getOperand(OpIdx); if (!Op.isReg()) { if (!is_contained(NonVecOpIndices, OpIdx)) return false; continue; } LLT Ty = MRI.getType(Op.getReg()); if (!Ty.isVector()) { if (!is_contained(NonVecOpIndices, OpIdx)) return false; continue; } if (Ty.getNumElements() != NumElts) return false; } return true; } #endif /// Fill \p DstOps with DstOps that have same number of elements combined as /// the Ty. These DstOps have either scalar type when \p NumElts = 1 or are /// vectors with \p NumElts elements. When Ty.getNumElements() is not multiple /// of \p NumElts last DstOp (leftover) has fewer then \p NumElts elements. static void makeDstOps(SmallVectorImpl<DstOp> &DstOps, LLT Ty, unsigned NumElts) { … } /// Operand \p Op is used on \p N sub-instructions. Fill \p Ops with \p N SrcOps /// made from \p Op depending on operand type. static void broadcastSrcOp(SmallVectorImpl<SrcOp> &Ops, unsigned N, MachineOperand &Op) { … } // Handle splitting vector operations which need to have the same number of // elements in each type index, but each type index may have a different element // type. // // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> -> // <2 x s64> = G_SHL <2 x s64>, <2 x s32> // <2 x s64> = G_SHL <2 x s64>, <2 x s32> // // Also handles some irregular breakdown cases, e.g. // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> -> // <2 x s64> = G_SHL <2 x s64>, <2 x s32> // s64 = G_SHL s64, s32 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorMultiEltType( GenericMachineInstr &MI, unsigned NumElts, std::initializer_list<unsigned> NonVecOpIndices) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI, unsigned NumElts) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT NarrowVecTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorShuffle( MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorReductions( MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorSeqReductions(MachineInstr &MI, unsigned int TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, LLT SrcTy, LLT NarrowTy, unsigned ScalarOpc) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, const LLT HalfTy, const LLT AmtTy) { … } // TODO: Optimize if constant shift amount. LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT RequestedTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy) { … } MachineInstrBuilder LegalizerHelper::getNeutralElementForVecReduce( unsigned Opcode, MachineIRBuilder &MIRBuilder, LLT Ty) { … } LegalizerHelper::LegalizeResult LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::equalizeVectorShuffleLengths(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::moreElementsVectorShuffle(MachineInstr &MI, unsigned int TypeIdx, LLT MoreTy) { … } void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs, ArrayRef<Register> Src1Regs, ArrayRef<Register> Src2Regs, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerBitCount(MachineInstr &MI) { … } // Check that (every element of) Reg is undef or not an exact multiple of BW. static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFunnelShiftWithInverse(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFunnelShiftAsShifts(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFunnelShift(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerEXT(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerTRUNC(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotateWithReverseRotate(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerRotate(MachineInstr &MI) { … } // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float // representation. LegalizerHelper::LegalizeResult LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) { … } // Expand s32 = G_UITOFP s64 to an IEEE float representation using bit // operations and G_SITOFP LegalizerHelper::LegalizeResult LegalizerHelper::lowerU64ToF32WithSITOFP(MachineInstr &MI) { … } // Expand s64 = G_UITOFP s64 using bit and float arithmetic operations to an // IEEE double representation. LegalizerHelper::LegalizeResult LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOINT_SAT(MachineInstr &MI) { … } // f64 -> f16 conversion using round-to-nearest-even rounding mode. LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) { … } static CmpInst::Predicate minMaxToCompare(unsigned Opc) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerThreewayCompare(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFCopySign(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFFloor(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMergeValues(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) { … } /// Lower a vector extract or insert by writing the vector to a stack temporary /// and reloading the element or vector. /// /// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx /// => /// %stack_temp = G_FRAME_INDEX /// G_STORE %vec, %stack_temp /// %idx = clamp(%idx, %vec.getNumElements()) /// %element_ptr = G_PTR_ADD %stack_temp, %idx /// %dst = G_LOAD %element_ptr LegalizerHelper::LegalizeResult LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerShuffleVector(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerVECTOR_COMPRESS(llvm::MachineInstr &MI) { … } Register LegalizerHelper::getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerStackSave(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerStackRestore(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerExtract(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerShlSat(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) { … } //{ (Src & Mask) >> N } | { (Src << N) & Mask } static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, const APInt &Mask) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerBitreverse(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerISFPCLASS(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerDIVREM(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerAbsToAddXor(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerFAbs(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerVectorReduction(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) { … } static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { … } // Returns a list of types to use for memory op lowering in MemOps. A partial // port of findOptimalMemOpLowering in TargetLowering. static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, const TargetLowering &TLI) { … } // Get a vectorized representation of the memset value operand, GISel edition. static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val, uint64_t KnownLen, Align Alignment, bool IsVolatile) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemcpyInline(MachineInstr &MI) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemcpyInline(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, Align DstAlign, Align SrcAlign, bool IsVolatile) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, uint64_t Limit, Align DstAlign, Align SrcAlign, bool IsVolatile) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, Align DstAlign, Align SrcAlign, bool IsVolatile) { … } LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { … }
Codebase Browser
llvm