//===-- HexagonVectorCombine.cpp ------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // HexagonVectorCombine is a utility class implementing a variety of functions // that assist in vector-based optimizations. // // AlignVectors: replace unaligned vector loads and stores with aligned ones. // HvxIdioms: recognize various opportunities to generate HVX intrinsic code. //===----------------------------------------------------------------------===// #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/InstSimplifyFolder.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsHexagon.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/PatternMatch.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/Local.h" #include "HexagonSubtarget.h" #include "HexagonTargetMachine.h" #include <algorithm> #include <deque> #include <map> #include <optional> #include <set> #include <utility> #include <vector> #define DEBUG_TYPE … usingnamespacellvm; namespace { cl::opt<bool> DumpModule("hvc-dump-module", cl::Hidden); cl::opt<bool> VAEnabled("hvc-va", cl::Hidden, cl::init(true)); // Align cl::opt<bool> VIEnabled("hvc-vi", cl::Hidden, cl::init(true)); // Idioms cl::opt<bool> VADoFullStores("hvc-va-full-stores", cl::Hidden); cl::opt<unsigned> VAGroupCountLimit("hvc-va-group-count-limit", cl::Hidden, cl::init(~0)); cl::opt<unsigned> VAGroupSizeLimit("hvc-va-group-size-limit", cl::Hidden, cl::init(~0)); class HexagonVectorCombine { … }; class AlignVectors { … }; LLVM_ATTRIBUTE_UNUSED raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::AddrInfo &AI) { … } LLVM_ATTRIBUTE_UNUSED raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::MoveGroup &MG) { … } LLVM_ATTRIBUTE_UNUSED raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::ByteSpan::Block &B) { … } LLVM_ATTRIBUTE_UNUSED raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::ByteSpan &BS) { … } class HvxIdioms { … }; [[maybe_unused]] raw_ostream &operator<<(raw_ostream &OS, const HvxIdioms::FxpOp &Op) { … } } // namespace namespace { template <typename T> T *getIfUnordered(T *MaybeT) { … } template <typename T> T *isCandidate(Instruction *In) { … } template <> LoadInst *isCandidate<LoadInst>(Instruction *In) { … } template <> StoreInst *isCandidate<StoreInst>(Instruction *In) { … } #if !defined(_MSC_VER) || _MSC_VER >= 1926 // VS2017 and some versions of VS2019 have trouble compiling this: // error C2976: 'std::map': too few template arguments // VS 2019 16.x is known to work, except for 16.4/16.5 (MSC_VER 1924/1925) template <typename Pred, typename... Ts> void erase_if(std::map<Ts...> &map, Pred p) #else template <typename Pred, typename T, typename U> void erase_if(std::map<T, U> &map, Pred p) #endif { … } // Forward other erase_ifs to the LLVM implementations. template <typename Pred, typename T> void erase_if(T &&container, Pred p) { … } } // namespace // --- Begin AlignVectors // For brevity, only consider loads. We identify a group of loads where we // know the relative differences between their addresses, so we know how they // are laid out in memory (relative to one another). These loads can overlap, // can be shorter or longer than the desired vector length. // Ultimately we want to generate a sequence of aligned loads that will load // every byte that the original loads loaded, and have the program use these // loaded values instead of the original loads. // We consider the contiguous memory area spanned by all these loads. // // Let's say that a single aligned vector load can load 16 bytes at a time. // If the program wanted to use a byte at offset 13 from the beginning of the // original span, it will be a byte at offset 13+x in the aligned data for // some x>=0. This may happen to be in the first aligned load, or in the load // following it. Since we generally don't know what the that alignment value // is at compile time, we proactively do valigns on the aligned loads, so that // byte that was at offset 13 is still at offset 13 after the valigns. // // This will be the starting point for making the rest of the program use the // data loaded by the new loads. // For each original load, and its users: // %v = load ... // ... = %v // ... = %v // we create // %new_v = extract/combine/shuffle data from loaded/valigned vectors so // it contains the same value as %v did before // then replace all users of %v with %new_v. // ... = %new_v // ... = %new_v auto AlignVectors::ByteSpan::extent() const -> int { … } auto AlignVectors::ByteSpan::section(int Start, int Length) const -> ByteSpan { … } auto AlignVectors::ByteSpan::shift(int Offset) -> ByteSpan & { … } auto AlignVectors::ByteSpan::values() const -> SmallVector<Value *, 8> { … } auto AlignVectors::getAlignFromValue(const Value *V) const -> Align { … } auto AlignVectors::getAddrInfo(Instruction &In) const -> std::optional<AddrInfo> { … } auto AlignVectors::isHvx(const AddrInfo &AI) const -> bool { … } auto AlignVectors::getPayload(Value *Val) const -> Value * { … } auto AlignVectors::getMask(Value *Val) const -> Value * { … } auto AlignVectors::getPassThrough(Value *Val) const -> Value * { … } auto AlignVectors::createAdjustedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy, int Adjust, const InstMap &CloneMap) const -> Value * { … } auto AlignVectors::createAlignedPointer(IRBuilderBase &Builder, Value *Ptr, Type *ValTy, int Alignment, const InstMap &CloneMap) const -> Value * { … } auto AlignVectors::createLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr, Value *Predicate, int Alignment, Value *Mask, Value *PassThru, ArrayRef<Value *> MDSources) const -> Value * { … } auto AlignVectors::createSimpleLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr, int Alignment, ArrayRef<Value *> MDSources) const -> Value * { … } auto AlignVectors::createPredicatedLoad(IRBuilderBase &Builder, Type *ValTy, Value *Ptr, Value *Predicate, int Alignment, ArrayRef<Value *> MDSources) const -> Value * { … } auto AlignVectors::createStore(IRBuilderBase &Builder, Value *Val, Value *Ptr, Value *Predicate, int Alignment, Value *Mask, ArrayRef<Value *> MDSources) const -> Value * { … } auto AlignVectors::createSimpleStore(IRBuilderBase &Builder, Value *Val, Value *Ptr, int Alignment, ArrayRef<Value *> MDSources) const -> Value * { … } auto AlignVectors::createPredicatedStore(IRBuilderBase &Builder, Value *Val, Value *Ptr, Value *Predicate, int Alignment, ArrayRef<Value *> MDSources) const -> Value * { … } auto AlignVectors::getUpwardDeps(Instruction *In, Instruction *Base) const -> DepList { … } auto AlignVectors::createAddressGroups() -> bool { … } auto AlignVectors::createLoadGroups(const AddrList &Group) const -> MoveList { … } auto AlignVectors::createStoreGroups(const AddrList &Group) const -> MoveList { … } auto AlignVectors::moveTogether(MoveGroup &Move) const -> bool { … } template <typename T> auto AlignVectors::cloneBefore(Instruction *To, T &&Insts) const -> InstMap { … } auto AlignVectors::realignLoadGroup(IRBuilderBase &Builder, const ByteSpan &VSpan, int ScLen, Value *AlignVal, Value *AlignAddr) const -> void { … } auto AlignVectors::realignStoreGroup(IRBuilderBase &Builder, const ByteSpan &VSpan, int ScLen, Value *AlignVal, Value *AlignAddr) const -> void { … } auto AlignVectors::realignGroup(const MoveGroup &Move) const -> bool { … } auto AlignVectors::makeTestIfUnaligned(IRBuilderBase &Builder, Value *AlignVal, int Alignment) const -> Value * { … } auto AlignVectors::isSectorTy(Type *Ty) const -> bool { … } auto AlignVectors::run() -> bool { … } // --- End AlignVectors // --- Begin HvxIdioms auto HvxIdioms::getNumSignificantBits(Value *V, Instruction *In) const -> std::pair<unsigned, Signedness> { … } auto HvxIdioms::canonSgn(SValue X, SValue Y) const -> std::pair<SValue, SValue> { … } // Match // (X * Y) [>> N], or // ((X * Y) + (1 << M)) >> N auto HvxIdioms::matchFxpMul(Instruction &In) const -> std::optional<FxpOp> { … } auto HvxIdioms::processFxpMul(Instruction &In, const FxpOp &Op) const -> Value * { … } auto HvxIdioms::processFxpMulChopped(IRBuilderBase &Builder, Instruction &In, const FxpOp &Op) const -> Value * { … } auto HvxIdioms::createMulQ15(IRBuilderBase &Builder, SValue X, SValue Y, bool Rounding) const -> Value * { … } auto HvxIdioms::createMulQ31(IRBuilderBase &Builder, SValue X, SValue Y, bool Rounding) const -> Value * { … } auto HvxIdioms::createAddCarry(IRBuilderBase &Builder, Value *X, Value *Y, Value *CarryIn) const -> std::pair<Value *, Value *> { … } auto HvxIdioms::createMul16(IRBuilderBase &Builder, SValue X, SValue Y) const -> Value * { … } auto HvxIdioms::createMulH16(IRBuilderBase &Builder, SValue X, SValue Y) const -> Value * { … } auto HvxIdioms::createMul32(IRBuilderBase &Builder, SValue X, SValue Y) const -> std::pair<Value *, Value *> { … } auto HvxIdioms::createAddLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX, ArrayRef<Value *> WordY) const -> SmallVector<Value *> { … } auto HvxIdioms::createMulLong(IRBuilderBase &Builder, ArrayRef<Value *> WordX, Signedness SgnX, ArrayRef<Value *> WordY, Signedness SgnY) const -> SmallVector<Value *> { … } auto HvxIdioms::run() -> bool { … } // --- End HvxIdioms auto HexagonVectorCombine::run() -> bool { … } auto HexagonVectorCombine::getIntTy(unsigned Width) const -> IntegerType * { … } auto HexagonVectorCombine::getByteTy(int ElemCount) const -> Type * { … } auto HexagonVectorCombine::getBoolTy(int ElemCount) const -> Type * { … } auto HexagonVectorCombine::getConstInt(int Val, unsigned Width) const -> ConstantInt * { … } auto HexagonVectorCombine::isZero(const Value *Val) const -> bool { … } auto HexagonVectorCombine::getIntValue(const Value *Val) const -> std::optional<APInt> { … } auto HexagonVectorCombine::isUndef(const Value *Val) const -> bool { … } auto HexagonVectorCombine::isTrue(const Value *Val) const -> bool { … } auto HexagonVectorCombine::isFalse(const Value *Val) const -> bool { … } auto HexagonVectorCombine::getHvxTy(Type *ElemTy, bool Pair) const -> VectorType * { … } auto HexagonVectorCombine::getSizeOf(const Value *Val, SizeKind Kind) const -> int { … } auto HexagonVectorCombine::getSizeOf(const Type *Ty, SizeKind Kind) const -> int { … } auto HexagonVectorCombine::getTypeAlignment(Type *Ty) const -> int { … } auto HexagonVectorCombine::length(Value *Val) const -> size_t { … } auto HexagonVectorCombine::length(Type *Ty) const -> size_t { … } auto HexagonVectorCombine::getNullValue(Type *Ty) const -> Constant * { … } auto HexagonVectorCombine::getFullValue(Type *Ty) const -> Constant * { … } auto HexagonVectorCombine::getConstSplat(Type *Ty, int Val) const -> Constant * { … } auto HexagonVectorCombine::simplify(Value *V) const -> Value * { … } // Insert bytes [Start..Start+Length) of Src into Dst at byte Where. auto HexagonVectorCombine::insertb(IRBuilderBase &Builder, Value *Dst, Value *Src, int Start, int Length, int Where) const -> Value * { … } auto HexagonVectorCombine::vlalignb(IRBuilderBase &Builder, Value *Lo, Value *Hi, Value *Amt) const -> Value * { … } auto HexagonVectorCombine::vralignb(IRBuilderBase &Builder, Value *Lo, Value *Hi, Value *Amt) const -> Value * { … } // Concatenates a sequence of vectors of the same type. auto HexagonVectorCombine::concat(IRBuilderBase &Builder, ArrayRef<Value *> Vecs) const -> Value * { … } auto HexagonVectorCombine::vresize(IRBuilderBase &Builder, Value *Val, int NewSize, Value *Pad) const -> Value * { … } auto HexagonVectorCombine::rescale(IRBuilderBase &Builder, Value *Mask, Type *FromTy, Type *ToTy) const -> Value * { … } // Bitcast to bytes, and return least significant bits. auto HexagonVectorCombine::vlsb(IRBuilderBase &Builder, Value *Val) const -> Value * { … } // Bitcast to bytes for non-bool. For bool, convert i1 -> i8. auto HexagonVectorCombine::vbytes(IRBuilderBase &Builder, Value *Val) const -> Value * { … } auto HexagonVectorCombine::subvector(IRBuilderBase &Builder, Value *Val, unsigned Start, unsigned Length) const -> Value * { … } auto HexagonVectorCombine::sublo(IRBuilderBase &Builder, Value *Val) const -> Value * { … } auto HexagonVectorCombine::subhi(IRBuilderBase &Builder, Value *Val) const -> Value * { … } auto HexagonVectorCombine::vdeal(IRBuilderBase &Builder, Value *Val0, Value *Val1) const -> Value * { … } auto HexagonVectorCombine::vshuff(IRBuilderBase &Builder, Value *Val0, Value *Val1) const -> Value * { … } auto HexagonVectorCombine::createHvxIntrinsic(IRBuilderBase &Builder, Intrinsic::ID IntID, Type *RetTy, ArrayRef<Value *> Args, ArrayRef<Type *> ArgTys, ArrayRef<Value *> MDSources) const -> Value * { … } auto HexagonVectorCombine::splitVectorElements(IRBuilderBase &Builder, Value *Vec, unsigned ToWidth) const -> SmallVector<Value *> { … } auto HexagonVectorCombine::joinVectorElements(IRBuilderBase &Builder, ArrayRef<Value *> Values, VectorType *ToType) const -> Value * { … } auto HexagonVectorCombine::calculatePointerDifference(Value *Ptr0, Value *Ptr1) const -> std::optional<int> { … } auto HexagonVectorCombine::getNumSignificantBits(const Value *V, const Instruction *CtxI) const -> unsigned { … } auto HexagonVectorCombine::getKnownBits(const Value *V, const Instruction *CtxI) const -> KnownBits { … } auto HexagonVectorCombine::isSafeToClone(const Instruction &In) const -> bool { … } template <typename T> auto HexagonVectorCombine::isSafeToMoveBeforeInBB(const Instruction &In, BasicBlock::const_iterator To, const T &IgnoreInsts) const -> bool { … } auto HexagonVectorCombine::isByteVecTy(Type *Ty) const -> bool { … } auto HexagonVectorCombine::getElementRange(IRBuilderBase &Builder, Value *Lo, Value *Hi, int Start, int Length) const -> Value * { … } // Pass management. namespace llvm { void initializeHexagonVectorCombineLegacyPass(PassRegistry &); FunctionPass *createHexagonVectorCombineLegacyPass(); } // namespace llvm namespace { class HexagonVectorCombineLegacy : public FunctionPass { … }; } // namespace char HexagonVectorCombineLegacy::ID = …; INITIALIZE_PASS_BEGIN(HexagonVectorCombineLegacy, DEBUG_TYPE, "Hexagon Vector Combine", false, false) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) INITIALIZE_PASS_END(HexagonVectorCombineLegacy, DEBUG_TYPE, "Hexagon Vector Combine", false, false) FunctionPass *llvm::createHexagonVectorCombineLegacyPass() { … }
Codebase Browser
llvm