CombinerHelper.cpp | Explore in Territory

//===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/Analysis/CmpInstAnalysis.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/LowLevelTypeUtils.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterBankInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/DivisionByConstantInfo.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include <cmath>
#include <optional>
#include <tuple>

#define DEBUG_TYPE …

usingnamespacellvm;
usingnamespaceMIPatternMatch;

// Option to allow testing of the combiner while no targets know about indexed
// addressing.
static cl::opt<bool>
    ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false),
                       cl::desc("Force all indexed operations to be "
                                "legal for the GlobalISel combiner"));

CombinerHelper::CombinerHelper(GISelChangeObserver &Observer,
                               MachineIRBuilder &B, bool IsPreLegalize,
                               GISelKnownBits *KB, MachineDominatorTree *MDT,
                               const LegalizerInfo *LI)
    : … { … }

const TargetLowering &CombinerHelper::getTargetLowering() const { … }

const MachineFunction &CombinerHelper::getMachineFunction() const { … }

const DataLayout &CombinerHelper::getDataLayout() const { … }

LLVMContext &CombinerHelper::getContext() const { … }

/// \returns The little endian in-memory byte position of byte \p I in a
/// \p ByteWidth bytes wide type.
///
/// E.g. Given a 4-byte type x, x[0] -> byte 0
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I) { … }

/// Determines the LogBase2 value for a non-null input value using the
/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
static Register buildLogBase2(Register V, MachineIRBuilder &MIB) { … }

/// \returns The big endian in-memory byte position of byte \p I in a
/// \p ByteWidth bytes wide type.
///
/// E.g. Given a 4-byte type x, x[0] -> byte 3
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) { … }

/// Given a map from byte offsets in memory to indices in a load/store,
/// determine if that map corresponds to a little or big endian byte pattern.
///
/// \param MemOffset2Idx maps memory offsets to address offsets.
/// \param LowestIdx is the lowest index in \p MemOffset2Idx.
///
/// \returns true if the map corresponds to a big endian byte pattern, false if
/// it corresponds to a little endian byte pattern, and std::nullopt otherwise.
///
/// E.g. given a 32-bit type x, and x[AddrOffset], the in-memory byte patterns
/// are as follows:
///
/// AddrOffset   Little endian    Big endian
/// 0            0                3
/// 1            1                2
/// 2            2                1
/// 3            3                0
static std::optional<bool>
isBigEndian(const SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
            int64_t LowestIdx) { … }

bool CombinerHelper::isPreLegalize() const { … }

bool CombinerHelper::isLegal(const LegalityQuery &Query) const { … }

bool CombinerHelper::isLegalOrBeforeLegalizer(
    const LegalityQuery &Query) const { … }

bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const { … }

void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg,
                                    Register ToReg) const { … }

void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI,
                                      MachineOperand &FromRegOp,
                                      Register ToReg) const { … }

void CombinerHelper::replaceOpcodeWith(MachineInstr &FromMI,
                                       unsigned ToOpcode) const { … }

const RegisterBank *CombinerHelper::getRegBank(Register Reg) const { … }

void CombinerHelper::setRegBank(Register Reg, const RegisterBank *RegBank) { … }

bool CombinerHelper::tryCombineCopy(MachineInstr &MI) { … }
bool CombinerHelper::matchCombineCopy(MachineInstr &MI) { … }
void CombinerHelper::applyCombineCopy(MachineInstr &MI) { … }

bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
    MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI,
                                               SmallVector<Register> &Ops) { … }
void CombinerHelper::applyCombineConcatVectors(MachineInstr &MI,
                                               SmallVector<Register> &Ops) { … }

bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
                                               SmallVector<Register> &Ops) { … }

void CombinerHelper::applyCombineShuffleConcat(MachineInstr &MI,
                                               SmallVector<Register> &Ops) { … }

bool CombinerHelper::tryCombineShuffleVector(MachineInstr &MI) { … }

bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI,
                                               SmallVectorImpl<Register> &Ops) { … }

void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI,
                                               const ArrayRef<Register> Ops) { … }

bool CombinerHelper::matchShuffleToExtract(MachineInstr &MI) { … }

void CombinerHelper::applyShuffleToExtract(MachineInstr &MI) { … }

namespace {

/// Select a preference between two uses. CurrentUse is the current preference
/// while *ForCandidate is attributes of the candidate under consideration.
PreferredTuple ChoosePreferredUse(MachineInstr &LoadMI,
                                  PreferredTuple &CurrentUse,
                                  const LLT TyForCandidate,
                                  unsigned OpcodeForCandidate,
                                  MachineInstr *MIForCandidate) { … }

/// Find a suitable place to insert some instructions and insert them. This
/// function accounts for special cases like inserting before a PHI node.
/// The current strategy for inserting before PHI's is to duplicate the
/// instructions for each predecessor. However, while that's ok for G_TRUNC
/// on most targets since it generally requires no code, other targets/cases may
/// want to try harder to find a dominating block.
static void InsertInsnsWithoutSideEffectsBeforeUse(
    MachineIRBuilder &Builder, MachineInstr &DefMI, MachineOperand &UseMO,
    std::function<void(MachineBasicBlock *, MachineBasicBlock::iterator,
                       MachineOperand &UseMO)>
        Inserter) { … }
} // end anonymous namespace

bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) { … }

static unsigned getExtLoadOpcForExtend(unsigned ExtOpc) { … }

bool CombinerHelper::matchCombineExtendingLoads(MachineInstr &MI,
                                                PreferredTuple &Preferred) { … }

void CombinerHelper::applyCombineExtendingLoads(MachineInstr &MI,
                                                PreferredTuple &Preferred) { … }

bool CombinerHelper::matchCombineLoadWithAndMask(MachineInstr &MI,
                                                 BuildFnTy &MatchInfo) { … }

bool CombinerHelper::isPredecessor(const MachineInstr &DefMI,
                                   const MachineInstr &UseMI) { … }

bool CombinerHelper::dominates(const MachineInstr &DefMI,
                               const MachineInstr &UseMI) { … }

bool CombinerHelper::matchSextTruncSextLoad(MachineInstr &MI) { … }

void CombinerHelper::applySextTruncSextLoad(MachineInstr &MI) { … }

bool CombinerHelper::matchSextInRegOfLoad(
    MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) { … }

void CombinerHelper::applySextInRegOfLoad(
    MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) { … }

/// Return true if 'MI' is a load or a store that may be fold it's address
/// operand into the load / store addressing mode.
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI,
                                    MachineRegisterInfo &MRI) { … }

static unsigned getIndexedOpc(unsigned LdStOpc) { … }

bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const { … }

static cl::opt<unsigned> PostIndexUseThreshold(
    "post-index-use-threshold", cl::Hidden, cl::init(32),
    cl::desc("Number of uses of a base pointer to check before it is no longer "
             "considered for post-indexing."));

bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,
                                            Register &Base, Register &Offset,
                                            bool &RematOffset) { … }

bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,
                                           Register &Base, Register &Offset) { … }

bool CombinerHelper::matchCombineExtractedVectorLoad(MachineInstr &MI,
                                                     BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchCombineIndexedLoadStore(
    MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { … }

void CombinerHelper::applyCombineIndexedLoadStore(
    MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { … }

bool CombinerHelper::matchCombineDivRem(MachineInstr &MI,
                                        MachineInstr *&OtherMI) { … }

void CombinerHelper::applyCombineDivRem(MachineInstr &MI,
                                        MachineInstr *&OtherMI) { … }

bool CombinerHelper::matchOptBrCondByInvertingCond(MachineInstr &MI,
                                                   MachineInstr *&BrCond) { … }

void CombinerHelper::applyOptBrCondByInvertingCond(MachineInstr &MI,
                                                   MachineInstr *&BrCond) { … }


bool CombinerHelper::tryEmitMemcpyInline(MachineInstr &MI) { … }

bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { … }

static APFloat constantFoldFpUnary(const MachineInstr &MI,
                                   const MachineRegisterInfo &MRI,
                                   const APFloat &Val) { … }

void CombinerHelper::applyCombineConstantFoldFpUnary(MachineInstr &MI,
                                                     const ConstantFP *Cst) { … }

bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
                                           PtrAddChain &MatchInfo) { … }

void CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
                                           PtrAddChain &MatchInfo) { … }

bool CombinerHelper::matchShiftImmedChain(MachineInstr &MI,
                                          RegisterImmPair &MatchInfo) { … }

void CombinerHelper::applyShiftImmedChain(MachineInstr &MI,
                                          RegisterImmPair &MatchInfo) { … }

bool CombinerHelper::matchShiftOfShiftedLogic(MachineInstr &MI,
                                              ShiftOfShiftedLogic &MatchInfo) { … }

void CombinerHelper::applyShiftOfShiftedLogic(MachineInstr &MI,
                                              ShiftOfShiftedLogic &MatchInfo) { … }

bool CombinerHelper::matchCommuteShift(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchCombineMulToShl(MachineInstr &MI,
                                          unsigned &ShiftVal) { … }

void CombinerHelper::applyCombineMulToShl(MachineInstr &MI,
                                          unsigned &ShiftVal) { … }

// shl ([sza]ext x), y => zext (shl x, y), if shift does not overflow source
bool CombinerHelper::matchCombineShlOfExtend(MachineInstr &MI,
                                             RegisterImmPair &MatchData) { … }

void CombinerHelper::applyCombineShlOfExtend(MachineInstr &MI,
                                             const RegisterImmPair &MatchData) { … }

bool CombinerHelper::matchCombineMergeUnmerge(MachineInstr &MI,
                                              Register &MatchInfo) { … }

static Register peekThroughBitcast(Register Reg,
                                   const MachineRegisterInfo &MRI) { … }

bool CombinerHelper::matchCombineUnmergeMergeToPlainValues(
    MachineInstr &MI, SmallVectorImpl<Register> &Operands) { … }

void CombinerHelper::applyCombineUnmergeMergeToPlainValues(
    MachineInstr &MI, SmallVectorImpl<Register> &Operands) { … }

bool CombinerHelper::matchCombineUnmergeConstant(MachineInstr &MI,
                                                 SmallVectorImpl<APInt> &Csts) { … }

void CombinerHelper::applyCombineUnmergeConstant(MachineInstr &MI,
                                                 SmallVectorImpl<APInt> &Csts) { … }

bool CombinerHelper::matchCombineUnmergeUndef(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) { … }

void CombinerHelper::applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) { … }

bool CombinerHelper::matchCombineUnmergeZExtToZExt(MachineInstr &MI) { … }

void CombinerHelper::applyCombineUnmergeZExtToZExt(MachineInstr &MI) { … }

bool CombinerHelper::matchCombineShiftToUnmerge(MachineInstr &MI,
                                                unsigned TargetShiftSize,
                                                unsigned &ShiftVal) { … }

void CombinerHelper::applyCombineShiftToUnmerge(MachineInstr &MI,
                                                const unsigned &ShiftVal) { … }

bool CombinerHelper::tryCombineShiftToUnmerge(MachineInstr &MI,
                                              unsigned TargetShiftAmount) { … }

bool CombinerHelper::matchCombineI2PToP2I(MachineInstr &MI, Register &Reg) { … }

void CombinerHelper::applyCombineI2PToP2I(MachineInstr &MI, Register &Reg) { … }

void CombinerHelper::applyCombineP2IToI2P(MachineInstr &MI, Register &Reg) { … }

bool CombinerHelper::matchCombineAddP2IToPtrAdd(
    MachineInstr &MI, std::pair<Register, bool> &PtrReg) { … }

void CombinerHelper::applyCombineAddP2IToPtrAdd(
    MachineInstr &MI, std::pair<Register, bool> &PtrReg) { … }

bool CombinerHelper::matchCombineConstPtrAddToI2P(MachineInstr &MI,
                                                  APInt &NewCst) { … }

void CombinerHelper::applyCombineConstPtrAddToI2P(MachineInstr &MI,
                                                  APInt &NewCst) { … }

bool CombinerHelper::matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg) { … }

bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI, Register &Reg) { … }

static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) { … }

bool CombinerHelper::matchCombineTruncOfShift(
    MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) { … }

void CombinerHelper::applyCombineTruncOfShift(
    MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) { … }

bool CombinerHelper::matchAnyExplicitUseIsUndef(MachineInstr &MI) { … }

bool CombinerHelper::matchAllExplicitUsesAreUndef(MachineInstr &MI) { … }

bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) { … }

bool CombinerHelper::matchUndefStore(MachineInstr &MI) { … }

bool CombinerHelper::matchUndefSelectCmp(MachineInstr &MI) { … }

bool CombinerHelper::matchInsertExtractVecEltOutOfBounds(MachineInstr &MI) { … }

bool CombinerHelper::matchConstantSelectCmp(MachineInstr &MI, unsigned &OpIdx) { … }

void CombinerHelper::eraseInst(MachineInstr &MI) { … }

bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1,
                                    const MachineOperand &MOP2) { … }

bool CombinerHelper::matchConstantOp(const MachineOperand &MOP, int64_t C) { … }

bool CombinerHelper::matchConstantFPOp(const MachineOperand &MOP, double C) { … }

void CombinerHelper::replaceSingleDefInstWithOperand(MachineInstr &MI,
                                                     unsigned OpIdx) { … }

void CombinerHelper::replaceSingleDefInstWithReg(MachineInstr &MI,
                                                 Register Replacement) { … }

bool CombinerHelper::matchConstantLargerBitWidth(MachineInstr &MI,
                                                 unsigned ConstIdx) { … }

void CombinerHelper::applyFunnelShiftConstantModulo(MachineInstr &MI) { … }

bool CombinerHelper::matchSelectSameVal(MachineInstr &MI) { … }

bool CombinerHelper::matchBinOpSameVal(MachineInstr &MI) { … }

bool CombinerHelper::matchOperandIsZero(MachineInstr &MI, unsigned OpIdx) { … }

bool CombinerHelper::matchOperandIsUndef(MachineInstr &MI, unsigned OpIdx) { … }

bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI,
                                                        unsigned OpIdx) { … }

void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI, double C) { … }

void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, int64_t C) { … }

void CombinerHelper::replaceInstWithConstant(MachineInstr &MI, APInt C) { … }

void CombinerHelper::replaceInstWithFConstant(MachineInstr &MI,
                                              ConstantFP *CFP) { … }

void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) { … }

bool CombinerHelper::matchSimplifyAddToSub(
    MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) { … }

bool CombinerHelper::matchCombineInsertVecElts(
    MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) { … }

void CombinerHelper::applyCombineInsertVecElts(
    MachineInstr &MI, SmallVectorImpl<Register> &MatchInfo) { … }

void CombinerHelper::applySimplifyAddToSub(
    MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) { … }

bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
    MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) { … }

void CombinerHelper::applyBuildInstructionSteps(
    MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) { … }

bool CombinerHelper::matchAshrShlToSextInreg(
    MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) { … }

void CombinerHelper::applyAshShlToSextInreg(
    MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) { … }

/// and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0
bool CombinerHelper::matchOverlappingAnd(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchRedundantAnd(MachineInstr &MI,
                                       Register &Replacement) { … }

bool CombinerHelper::matchRedundantOr(MachineInstr &MI, Register &Replacement) { … }

bool CombinerHelper::matchRedundantSExtInReg(MachineInstr &MI) { … }

static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
                             int64_t Cst, bool IsVector, bool IsFP) { … }

// This combine tries to reduce the number of scalarised G_TRUNC instructions by
// using vector truncates instead
//
// EXAMPLE:
// %a(i32), %b(i32) = G_UNMERGE_VALUES %src(<2 x i32>)
// %T_a(i16) = G_TRUNC %a(i32)
// %T_b(i16) = G_TRUNC %b(i32)
// %Undef(i16) = G_IMPLICIT_DEF(i16)
// %dst(v4i16) = G_BUILD_VECTORS %T_a(i16), %T_b(i16), %Undef(i16), %Undef(i16)
//
// ===>
// %Undef(<2 x i32>) = G_IMPLICIT_DEF(<2 x i32>)
// %Mid(<4 x s32>) = G_CONCAT_VECTORS %src(<2 x i32>), %Undef(<2 x i32>)
// %dst(<4 x s16>) = G_TRUNC %Mid(<4 x s32>)
//
// Only matches sources made up of G_TRUNCs followed by G_IMPLICIT_DEFs
bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
                                            Register &MatchInfo) { … }

void CombinerHelper::applyUseVectorTruncate(MachineInstr &MI,
                                            Register &MatchInfo) { … }

bool CombinerHelper::matchNotCmp(MachineInstr &MI,
                                 SmallVectorImpl<Register> &RegsToNegate) { … }

void CombinerHelper::applyNotCmp(MachineInstr &MI,
                                 SmallVectorImpl<Register> &RegsToNegate) { … }

bool CombinerHelper::matchXorOfAndWithSameReg(
    MachineInstr &MI, std::pair<Register, Register> &MatchInfo) { … }

void CombinerHelper::applyXorOfAndWithSameReg(
    MachineInstr &MI, std::pair<Register, Register> &MatchInfo) { … }

bool CombinerHelper::matchPtrAddZero(MachineInstr &MI) { … }

void CombinerHelper::applyPtrAddZero(MachineInstr &MI) { … }

/// The second source operand is known to be a power of 2.
void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) { … }

bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI,
                                              unsigned &SelectOpNo) { … }

/// \p SelectOperand is the operand in binary operator \p MI that is the select
/// to fold.
void CombinerHelper::applyFoldBinOpIntoSelect(MachineInstr &MI,
                                              const unsigned &SelectOperand) { … }

std::optional<SmallVector<Register, 8>>
CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const { … }

/// Helper function for findLoadOffsetsForLoadOrCombine.
///
/// Check if \p Reg is the result of loading a \p MemSizeInBits wide value,
/// and then moving that value into a specific byte offset.
///
/// e.g. x[i] << 24
///
/// \returns The load instruction and the byte offset it is moved into.
static std::optional<std::pair<GZExtLoad *, int64_t>>
matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits,
                         const MachineRegisterInfo &MRI) { … }

std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
CombinerHelper::findLoadOffsetsForLoadOrCombine(
    SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx,
    const SmallVector<Register, 8> &RegsToVisit, const unsigned MemSizeInBits) { … }

bool CombinerHelper::matchLoadOrCombine(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchExtendThroughPhis(MachineInstr &MI,
                                            MachineInstr *&ExtMI) { … }

void CombinerHelper::applyExtendThroughPhis(MachineInstr &MI,
                                            MachineInstr *&ExtMI) { … }

bool CombinerHelper::matchExtractVecEltBuildVec(MachineInstr &MI,
                                                Register &Reg) { … }

void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI,
                                                Register &Reg) { … }

bool CombinerHelper::matchExtractAllEltsFromBuildVector(
    MachineInstr &MI,
    SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) { … }

void CombinerHelper::applyExtractAllEltsFromBuildVector(
    MachineInstr &MI,
    SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) { … }

void CombinerHelper::applyBuildFn(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

void CombinerHelper::applyBuildFnNoErase(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI,
                                               BuildFnTy &MatchInfo) { … }

/// Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
bool CombinerHelper::matchFunnelShiftToRotate(MachineInstr &MI) { … }

void CombinerHelper::applyFunnelShiftToRotate(MachineInstr &MI) { … }

// Fold (rot x, c) -> (rot x, c % BitSize)
bool CombinerHelper::matchRotateOutOfRange(MachineInstr &MI) { … }

void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) { … }

bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
                                                   int64_t &MatchInfo) { … }

bool CombinerHelper::matchICmpToLHSKnownBits(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

// Replace (and (or x, c1), c2) with (and x, c2) iff c1 & c2 == 0
bool CombinerHelper::matchAndOrDisjointMask(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

/// Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
bool CombinerHelper::matchBitfieldExtractFromSExtInReg(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

/// Form a G_UBFX from "(a srl b) & mask", where b and mask are constants.
bool CombinerHelper::matchBitfieldExtractFromAnd(MachineInstr &MI,
                                                 BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchBitfieldExtractFromShr(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchBitfieldExtractFromShrAnd(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::reassociationCanBreakAddressingModePattern(
    MachineInstr &MI) { … }

bool CombinerHelper::matchReassocConstantInnerRHS(GPtrAdd &MI,
                                                  MachineInstr *RHS,
                                                  BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
                                                  MachineInstr *LHS,
                                                  MachineInstr *RHS,
                                                  BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchReassocFoldConstantsInSubTree(GPtrAdd &MI,
                                                        MachineInstr *LHS,
                                                        MachineInstr *RHS,
                                                        BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchReassocPtrAdd(MachineInstr &MI,
                                        BuildFnTy &MatchInfo) { … }
bool CombinerHelper::tryReassocBinOp(unsigned Opc, Register DstReg,
                                     Register OpLHS, Register OpRHS,
                                     BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
                                           BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI, APInt &MatchInfo) { … }

bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI, APInt &MatchInfo) { … }

bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI, ConstantFP* &MatchInfo) { … }

bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
                                          ConstantFP *&MatchInfo) { … }

bool CombinerHelper::matchNarrowBinopFeedingAnd(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchAddEToAddO(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
                                        BuildFnTy &MatchInfo) { … }

MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) { … }

bool CombinerHelper::matchUDivByConst(MachineInstr &MI) { … }

void CombinerHelper::applyUDivByConst(MachineInstr &MI) { … }

bool CombinerHelper::matchSDivByConst(MachineInstr &MI) { … }

void CombinerHelper::applySDivByConst(MachineInstr &MI) { … }

MachineInstr *CombinerHelper::buildSDivUsingMul(MachineInstr &MI) { … }

bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) { … }

void CombinerHelper::applySDivByPow2(MachineInstr &MI) { … }

void CombinerHelper::applyUDivByPow2(MachineInstr &MI) { … }

bool CombinerHelper::matchUMulHToLShr(MachineInstr &MI) { … }

void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) { … }

bool CombinerHelper::matchRedundantNegOperands(MachineInstr &MI,
                                               BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchFsubToFneg(MachineInstr &MI, Register &MatchInfo) { … }

void CombinerHelper::applyFsubToFneg(MachineInstr &MI, Register &MatchInfo) { … }

/// Checks if \p MI is TargetOpcode::G_FMUL and contractable either
/// due to global flags or MachineInstr flags.
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally) { … }

static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1,
                        const MachineRegisterInfo &MRI) { … }

bool CombinerHelper::canCombineFMadOrFMA(MachineInstr &MI,
                                         bool &AllowFusionGlobally,
                                         bool &HasFMAD, bool &Aggressive,
                                         bool CanReassociate) { … }

bool CombinerHelper::matchCombineFAddFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFAddFMAFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFSubFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFSubFNegFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFSubFpExtFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(
    MachineInstr &MI, std::function<void(MachineIRBuilder &)> &MatchInfo) { … }

bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI,
                                            unsigned &IdxToPropagate) { … }

bool CombinerHelper::matchAddSubSameReg(MachineInstr &MI, Register &Src) { … }

bool CombinerHelper::matchBuildVectorIdentityFold(MachineInstr &MI,
                                                  Register &MatchInfo) { … }

bool CombinerHelper::matchTruncBuildVectorFold(MachineInstr &MI,
                                               Register &MatchInfo) { … }

bool CombinerHelper::matchTruncLshrBuildVectorFold(MachineInstr &MI,
                                                   Register &MatchInfo) { … }

unsigned CombinerHelper::getFPMinMaxOpcForSelect(
    CmpInst::Predicate Pred, LLT DstTy,
    SelectPatternNaNBehaviour VsNaNRetVal) const { … }

CombinerHelper::SelectPatternNaNBehaviour
CombinerHelper::computeRetValAgainstNaN(Register LHS, Register RHS,
                                        bool IsOrderedComparison) const { … }

bool CombinerHelper::matchFPSelectToMinMax(Register Dst, Register Cond,
                                           Register TrueVal, Register FalseVal,
                                           BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchSimplifySelectToMinMax(MachineInstr &MI,
                                                 BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchRedundantBinOpInEquality(MachineInstr &MI,
                                                   BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI) { … }

bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) { … }

bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) { … }

void CombinerHelper::applyCommuteBinOpOperands(MachineInstr &MI) { … }

bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) { … }

bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) { … }

// Ignores COPYs during conformance checks.
// FIXME scalable vectors.
bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,
                                           bool AllowUndefs) { … }

// Ignores COPYs during lookups.
// FIXME scalable vectors
std::optional<APInt>
CombinerHelper::getConstantOrConstantSplatVector(Register Src) { … }

// FIXME G_SPLAT_VECTOR
bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const { … }

// TODO: use knownbits to determine zeros
bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,
                                              BuildFnTy &MatchInfo) { … }

// TODO: use knownbits to determine zeros
bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,
                                              BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchSelectIMinMax(const MachineOperand &MO,
                                        BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

/// Fold (icmp Pred1 V1, C1) && (icmp Pred2 V2, C2)
/// or   (icmp Pred1 V1, C1) || (icmp Pred2 V2, C2)
/// into a single comparison using range-based reasoning.
/// see InstCombinerImpl::foldAndOrOfICmpsUsingRanges.
bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(GLogicalBinOp *Logic,
                                                    BuildFnTy &MatchInfo) { … }

bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,
                                         BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) { … }

void CombinerHelper::applyBuildFnMO(const MachineOperand &MO,
                                    BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchFPowIExpansion(MachineInstr &MI, int64_t Exponent) { … }

void CombinerHelper::applyExpandFPowI(MachineInstr &MI, int64_t Exponent) { … }

bool CombinerHelper::matchFoldAPlusC1MinusC2(const MachineInstr &MI,
                                             BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchFoldC2MinusAPlusC1(const MachineInstr &MI,
                                             BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchFoldAMinusC1MinusC2(const MachineInstr &MI,
                                              BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchFoldC1Minus2MinusC2(const MachineInstr &MI,
                                              BuildFnTy &MatchInfo) { … }

bool CombinerHelper::matchFoldAMinusC1PlusC2(const MachineInstr &MI,
                                             BuildFnTy &MatchInfo) { … }
llvm/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
