LegalizeVectorOps.cpp | Explore in Territory

//===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the SelectionDAG::LegalizeVectors method.
//
// The vector legalizer looks for vector operations which might need to be
// scalarized and legalizes them. This is a separate step from Legalize because
// scalarizing can introduce illegal types.  For example, suppose we have an
// ISD::SDIV of type v2i64 on x86-32.  The type is legal (for example, addition
// on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
// operation, which introduces nodes with the illegal type i64 which must be
// expanded.  Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
// the operation must be unrolled, which introduces nodes with the illegal
// type i8 which must be promoted.
//
// This does not legalize vector manipulations like ISD::BUILD_VECTOR,
// or operations that happen to take a vector which are custom-lowered;
// the legalization for such operations never produces nodes
// with illegal types, so it's okay to put off legalizing them until
// SelectionDAG::Legalize runs.
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include <cassert>
#include <cstdint>
#include <iterator>
#include <utility>

usingnamespacellvm;

#define DEBUG_TYPE …

namespace {

class VectorLegalizer { … };

} // end anonymous namespace

bool VectorLegalizer::Run() { … }

SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) { … }

SDValue
VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
                                            MutableArrayRef<SDValue> Results) { … }

SDValue VectorLegalizer::LegalizeOp(SDValue Op) { … }

// FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
// merge them somehow?
bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
                                            SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::PromoteSETCC(SDNode *Node,
                                   SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::PromoteSTRICT(SDNode *Node,
                                    SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
                                       SmallVectorImpl<SDValue> &Results) { … }

// For FP_TO_INT we promote the result type to a vector type with wider
// elements and then truncate the result.  This is different from the default
// PromoteVector which uses bitcast to promote thus assumning that the
// promoted vector type has the same overall size.
void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
                                       SmallVectorImpl<SDValue> &Results) { … }

std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) { … }

SDValue VectorLegalizer::ExpandStore(SDNode *N) { … }

void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) { … }

SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) { … }

// Generically expand a vector anyext in register to a shuffle of the relevant
// lanes into the appropriate locations, with other lanes left undef.
SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) { … }

// Generically expand a vector zext in register to a shuffle of the relevant
// lanes into the appropriate locations, a blend of zero into the high bits,
// and a bitcast to the wider element type.
SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) { … }

static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) { … }

SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) { … }

void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
                                       SmallVectorImpl<SDValue> &Results) { … }

SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandVP_SELECT(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandVP_MERGE(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandVP_REM(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandVP_FNEG(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandVP_FABS(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandVP_FCOPYSIGN(SDNode *Node) { … }

void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
                                       SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
                                          SmallVectorImpl<SDValue> &Results) { … }

SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandFABS(SDNode *Node) { … }

SDValue VectorLegalizer::ExpandFCOPYSIGN(SDNode *Node) { … }

void VectorLegalizer::ExpandFSUB(SDNode *Node,
                                 SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandSETCC(SDNode *Node,
                                  SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
                                     SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
                                     SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandMULO(SDNode *Node,
                                 SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
                                          SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
                                       SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::ExpandREM(SDNode *Node,
                                SmallVectorImpl<SDValue> &Results) { … }

// Try to expand libm nodes into vector math routine calls. Callers provide the
// LibFunc equivalent of the passed in Node, which is used to lookup mappings
// within TargetLibraryInfo. The only mappings considered are those where the
// result and all operands are the same vector type. While predicated nodes are
// not supported, we will emit calls to masked routines by passing in an all
// true mask.
bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
                                           SmallVectorImpl<SDValue> &Results) { … }

/// Try to expand the node to a vector libcall based on the result type.
bool VectorLegalizer::tryExpandVecMathCall(
    SDNode *Node, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
    RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
    RTLIB::Libcall Call_PPCF128, SmallVectorImpl<SDValue> &Results) { … }

void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
                                       SmallVectorImpl<SDValue> &Results) { … }

SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) { … }

bool SelectionDAG::LegalizeVectors() { … }
llvm/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
