#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
#include "llvm/Transforms/Vectorize/LoopVectorize.h"
usingnamespacellvm;
usingnamespacePatternMatch;
#define LV_NAME …
#define DEBUG_TYPE …
static cl::opt<bool>
EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden,
cl::desc("Enable if-conversion during vectorization."));
static cl::opt<bool>
AllowStridedPointerIVs("lv-strided-pointer-ivs", cl::init(false), cl::Hidden,
cl::desc("Enable recognition of non-constant strided "
"pointer induction variables."));
namespace llvm {
cl::opt<bool>
HintsAllowReordering("hints-allow-reordering", cl::init(true), cl::Hidden,
cl::desc("Allow enabling loop hints to reorder "
"FP operations during vectorization."));
}
static cl::opt<unsigned> VectorizeSCEVCheckThreshold(
"vectorize-scev-check-threshold", cl::init(16), cl::Hidden,
cl::desc("The maximum number of SCEV checks allowed."));
static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
"pragma-vectorize-scev-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum number of SCEV checks allowed with a "
"vectorize(enable) pragma"));
static cl::opt<LoopVectorizeHints::ScalableForceKind>
ForceScalableVectorization(
"scalable-vectorization", cl::init(LoopVectorizeHints::SK_Unspecified),
cl::Hidden,
cl::desc("Control whether the compiler can use scalable vectors to "
"vectorize a loop"),
cl::values(
clEnumValN(LoopVectorizeHints::SK_FixedWidthOnly, "off",
"Scalable vectorization is disabled."),
clEnumValN(
LoopVectorizeHints::SK_PreferScalable, "preferred",
"Scalable vectorization is available and favored when the "
"cost is inconclusive."),
clEnumValN(
LoopVectorizeHints::SK_PreferScalable, "on",
"Scalable vectorization is available and favored when the "
"cost is inconclusive.")));
static const unsigned MaxInterleaveFactor = …;
namespace llvm {
bool LoopVectorizeHints::Hint::validate(unsigned Val) { … }
LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
bool InterleaveOnlyWhenForced,
OptimizationRemarkEmitter &ORE,
const TargetTransformInfo *TTI)
: … { … }
void LoopVectorizeHints::setAlreadyVectorized() { … }
bool LoopVectorizeHints::allowVectorization(
Function *F, Loop *L, bool VectorizeOnlyWhenForced) const { … }
void LoopVectorizeHints::emitRemarkWithHints() const { … }
const char *LoopVectorizeHints::vectorizeAnalysisPassName() const { … }
bool LoopVectorizeHints::allowReordering() const { … }
void LoopVectorizeHints::getHintsFromMetadata() { … }
void LoopVectorizeHints::setHint(StringRef Name, Metadata *Arg) { … }
static bool isUniformLoop(Loop *Lp, Loop *OuterLp) { … }
static bool isUniformLoopNest(Loop *Lp, Loop *OuterLp) { … }
static Type *convertPointerToIntegerType(const DataLayout &DL, Type *Ty) { … }
static Type *getWiderType(const DataLayout &DL, Type *Ty0, Type *Ty1) { … }
static bool hasOutsideLoopUser(const Loop *TheLoop, Instruction *Inst,
SmallPtrSetImpl<Value *> &AllowedExit) { … }
static bool storeToSameAddress(ScalarEvolution *SE, StoreInst *A,
StoreInst *B) { … }
int LoopVectorizationLegality::isConsecutivePtr(Type *AccessTy,
Value *Ptr) const { … }
bool LoopVectorizationLegality::isInvariant(Value *V) const { … }
namespace {
class SCEVAddRecForUniformityRewriter
: public SCEVRewriteVisitor<SCEVAddRecForUniformityRewriter> { … };
}
bool LoopVectorizationLegality::isUniform(Value *V, ElementCount VF) const { … }
bool LoopVectorizationLegality::isUniformMemOp(Instruction &I,
ElementCount VF) const { … }
bool LoopVectorizationLegality::canVectorizeOuterLoop() { … }
void LoopVectorizationLegality::addInductionPhi(
PHINode *Phi, const InductionDescriptor &ID,
SmallPtrSetImpl<Value *> &AllowedExit) { … }
bool LoopVectorizationLegality::setupOuterLoopInductions() { … }
static bool isTLIScalarize(const TargetLibraryInfo &TLI, const CallInst &CI) { … }
bool LoopVectorizationLegality::canVectorizeInstrs() { … }
bool LoopVectorizationLegality::canVectorizeMemory() { … }
bool LoopVectorizationLegality::canVectorizeFPMath(
bool EnableStrictReductions) { … }
bool LoopVectorizationLegality::isInvariantStoreOfReduction(StoreInst *SI) { … }
bool LoopVectorizationLegality::isInvariantAddressOfReduction(Value *V) { … }
bool LoopVectorizationLegality::isInductionPhi(const Value *V) const { … }
const InductionDescriptor *
LoopVectorizationLegality::getIntOrFpInductionDescriptor(PHINode *Phi) const { … }
const InductionDescriptor *
LoopVectorizationLegality::getPointerInductionDescriptor(PHINode *Phi) const { … }
bool LoopVectorizationLegality::isCastedInductionVariable(
const Value *V) const { … }
bool LoopVectorizationLegality::isInductionVariable(const Value *V) const { … }
bool LoopVectorizationLegality::isFixedOrderRecurrence(
const PHINode *Phi) const { … }
bool LoopVectorizationLegality::blockNeedsPredication(BasicBlock *BB) const { … }
bool LoopVectorizationLegality::blockCanBePredicated(
BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs,
SmallPtrSetImpl<const Instruction *> &MaskedOp) const { … }
bool LoopVectorizationLegality::canVectorizeWithIfConvert() { … }
bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
bool UseVPlanNativePath) { … }
bool LoopVectorizationLegality::canVectorizeLoopNestCFG(
Loop *Lp, bool UseVPlanNativePath) { … }
bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) { … }
bool LoopVectorizationLegality::canFoldTailByMasking() const { … }
void LoopVectorizationLegality::prepareToFoldTailByMasking() { … }
}