//===- LinalgTransformOps.cpp - Implementation of Linalg transform ops ----===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h" #include "mlir/AsmParser/AsmParser.h" #include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Arith/Utils/Utils.h" #include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h" #include "mlir/Dialect/GPU/IR/GPUDialect.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/Linalg/TransformOps/GPUHeuristics.h" #include "mlir/Dialect/Linalg/TransformOps/Syntax.h" #include "mlir/Dialect/Linalg/Transforms/Hoisting.h" #include "mlir/Dialect/Linalg/Transforms/Transforms.h" #include "mlir/Dialect/Linalg/Utils/Utils.h" #include "mlir/Dialect/SCF/Transforms/TileUsingInterface.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/Dialect/Tensor/Utils/Utils.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Utils/Utils.h" #include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/Dialect/Utils/StaticValueUtils.h" #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" #include "mlir/IR/BuiltinTypeInterfaces.h" #include "mlir/IR/PatternMatch.h" #include "mlir/IR/TypeUtilities.h" #include "mlir/Interfaces/TilingInterface.h" #include "mlir/Support/LLVM.h" #include "mlir/Support/TypeID.h" #include "mlir/Transforms/GreedyPatternRewriteDriver.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/Debug.h" #include <type_traits> usingnamespacemlir; usingnamespacemlir::linalg; usingnamespacemlir::transform; #define DEBUG_TYPE … #define DBGS() … #define DBGSNL() … #define LDBG(X) … /// Attempts to apply the pattern specified as template argument to the given /// operation. The pattern is expected to have a `returningMatchAndRewrite` /// function that returns the "main" result or failure. Returns failure if the /// pattern failed to apply. Extra arguments are forwarded to the pattern /// constructor. template <typename PatternTy, typename... Args> static FailureOr<LinalgOp> tryApply(Operation *operation, Args &&...args) { … } /// Assuming that `ofr` is an index attr or a param of index type /// or a transform dialect handle mapped to exactly one op /// with one index result, return that value. static DiagnosedSilenceableFailure unpackSingleIndexResultPayloadOperations( transform::TransformState &state, TransformOpInterface transformOp, SmallVector<OpFoldResult> &result, ArrayRef<OpFoldResult> ofrs) { … } // Given a list of params that are index attrs or a list of OpFoldResults // that are either index attrs or op handles, return a list of OpFoldResults // of index attrs or a list of OpFoldResults where all op handles are // replaced with the first (and only) OpResult of that payload op. // (There must be exactly one parameter associated with the AnyParamType or // one mapped payload op which must have exactly one index result.) static DiagnosedSilenceableFailure unpackSingleIndexResultPayloadOperations( transform::TransformState &state, TransformOpInterface transformOp, SmallVector<OpFoldResult> &result, Value packedHandle) { … } /// When possible, converts each `OpFoldResult` in `mixedResult` to /// an integer if the value can be statically inferred. If a result /// is a `Value` then it must be either a `ParamType` or a handle /// to an a constant like op. static DiagnosedSilenceableFailure reifyMixedParamAndHandleResults( TransformState &state, TransformOpInterface &transformOp, ArrayRef<OpFoldResult> mixedResults, SmallVectorImpl<int64_t> &reified) { … } //===----------------------------------------------------------------------===// // Apply...PatternsOp //===----------------------------------------------------------------------===// void transform::ApplyEraseUnnecessaryInputsPatternsOp::populatePatterns( RewritePatternSet &patterns) { … } void transform::ApplyFoldUnitExtentDimsViaReshapesPatternsOp::populatePatterns( RewritePatternSet &patterns) { … } void transform::ApplyFoldUnitExtentDimsViaSlicesPatternsOp::populatePatterns( RewritePatternSet &patterns) { … } void transform::ApplyTilingCanonicalizationPatternsOp::populatePatterns( RewritePatternSet &patterns) { … } void transform::ApplyFoldAddIntoDestPatternsOp::populatePatterns( RewritePatternSet &patterns) { … } //===----------------------------------------------------------------------===// // BufferizeToAllocationOp //===----------------------------------------------------------------------===// void transform::BufferizeToAllocationOp::build(OpBuilder &b, OperationState &result, Value target, Attribute memorySpace) { … } void transform::BufferizeToAllocationOp::build(OpBuilder &b, OperationState &result, Value target, int64_t memorySpace) { … } namespace { class NewOpsListener : public RewriterBase::ForwardingListener { … }; } // namespace DiagnosedSilenceableFailure transform::BufferizeToAllocationOp::apply( transform::TransformRewriter &rewriter, transform::TransformResults &results, transform::TransformState &state) { … } void transform::BufferizeToAllocationOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } LogicalResult transform::BufferizeToAllocationOp::verify() { … } //===----------------------------------------------------------------------===// // DecomposeOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::DecomposeOp::applyToOne(transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // DecomposeInterfaceOp //===----------------------------------------------------------------------===// // Decompose the target operation if it implements the AggregatedOpInterface. // Push the decomposed operations (the ones that replaces the values produced by // \p target) in the `results`. DiagnosedSilenceableFailure transform::DecomposeInterfaceOp::applyToOne( transform::TransformRewriter &rewriter, Operation *target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // EliminateLinalgOpAnchoredEmptyTensorsOp //===----------------------------------------------------------------------===// void transform::EliminateLinalgOpAnchoredEmptyTensorsOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } DiagnosedSilenceableFailure transform::EliminateLinalgOpAnchoredEmptyTensorsOp::apply( transform::TransformRewriter &rewriter, TransformResults &transformResults, TransformState &state) { … } //===----------------------------------------------------------------------===// // FuseOp //===----------------------------------------------------------------------===// /// Apply a tiling transformation to all payload ops and store both the /// tiled operation as well as the created tile loops. template <typename Range> static LogicalResult applyTilingToAll( RewriterBase &rewriter, Operation *transformOp, Range &&payloadOps, unsigned numLoops, transform::TransformResults &transformResults, function_ref<FailureOr<scf::SCFTileAndFuseResult>(TilingInterface)> applyFn) { … } DiagnosedSilenceableFailure transform::FuseOp::apply(transform::TransformRewriter &rewriter, mlir::transform::TransformResults &transformResults, mlir::transform::TransformState &state) { … } LogicalResult transform::FuseOp::verify() { … } //===----------------------------------------------------------------------===// // FuseIntoContainingOp //===----------------------------------------------------------------------===// void transform::FuseIntoContainingOp::build(OpBuilder &builder, OperationState &result, Value producerOp, Value containingOp) { … } /// Add new operands to the forall op for users of the producerOp /// that are dominated by the containing scf.forall op. static Operation *replaceForAllWithNewSignature( RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp, Operation *containingOp, TilingResult &tileAndFuseResult, int64_t resultNumber, SmallVector<OpFoldResult> &offsets, SmallVector<OpFoldResult> &sizes) { … } /// Find the first "extract" user of `producerOp` and tile it right before its /// use. The tiled op is fused under the `containingOp`. /// Return this fused op on success or nullptr if anything fails. /// If tiled op has uses that are dominated by `containingOp`, return /// a new `containingOp` with results of the fused op appended to /// results of the `containingOp` or nullptr if there are no dominated uses. static std::tuple<SmallVector<Operation *>, Operation *> tileAndFuseFirstExtractUse(RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp, Operation *containingOp) { … } /// First, find the first "scf::ForallOp" user of `producerOp` and ensure /// it is exactly the `containingOp`, otherwise bail. /// Then, find the first "extract" user of the tied block argument and tile it /// right before its "extract" use. The tiled op is fused under the /// `containingOp`. /// Return this fused op on success or nullptr if anything fails. static SmallVector<Operation *> tileAndFuseFirstExtractUseThroughContainingOpBlockArgument( RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp, Operation *containingOp) { … } static Operation *cloneAndFuseFirstUse(RewriterBase &rewriter, Diagnostic &diag, Operation *producerOp, Operation *containingOp) { … } bool transform::FuseIntoContainingOp::allowsRepeatedHandleOperands() { … } DiagnosedSilenceableFailure transform::FuseIntoContainingOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &results, transform::TransformState &state) { … } void transform::FuseIntoContainingOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } //===----------------------------------------------------------------------===// // GeneralizeOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::GeneralizeOp::applyToOne(transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // SpecializeOp //===----------------------------------------------------------------------===/ DiagnosedSilenceableFailure transform::SpecializeOp::applyToOne(transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // InterchangeOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::InterchangeOp::applyToOne(transform::TransformRewriter &rewriter, GenericOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } LogicalResult transform::InterchangeOp::verify() { … } //===----------------------------------------------------------------------===// // LowerPackOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::LowerPackOp::applyToOne( transform::TransformRewriter &rewriter, tensor::PackOp target, transform::ApplyToEachResultList &transformResults, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // LowerUnPackOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::LowerUnPackOp::applyToOne( transform::TransformRewriter &rewriter, tensor::UnPackOp target, transform::ApplyToEachResultList &transformResults, transform::TransformState &state) { … } //===---------------------------------------------------------------------===// // MatchOp //===---------------------------------------------------------------------===// void transform::MatchOp::build(OpBuilder &builder, OperationState &result, Value target, ArrayRef<StringRef> opNames) { … } void transform::MatchOp::build(OpBuilder &builder, OperationState &result, TypeRange resultTypes, Value target, ArrayRef<StringRef> opNames) { … } DiagnosedSilenceableFailure transform::MatchOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &results, transform::TransformState &state) { … } //===---------------------------------------------------------------------===// // MultiTileSizesOp //===---------------------------------------------------------------------===// static void printMultitileSizesTypes(OpAsmPrinter &printer, Operation *op, Type targetType, Type lowSizeType, Type, Type) { … } static ParseResult parseMultitileSizesTypes(OpAsmParser &parser, Type &targetType, Type &lowSizeType, Type &highSizeType, Type &splitPointType) { … } DiagnosedSilenceableFailure transform::MultiTileSizesOp::applyToOne( transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, TransformState &state) { … } void transform::MultiTileSizesOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } LogicalResult transform::MultiTileSizesOp::verify() { … } //===---------------------------------------------------------------------===// // PackOp //===---------------------------------------------------------------------===// void transform::PackOp::build(OpBuilder &builder, OperationState &result, Value target, ArrayRef<OpFoldResult> mixedPackedSizes) { … } SmallVector<OpFoldResult> transform::PackOp::getMixedPackedSizes() { … } DiagnosedSilenceableFailure transform::PackOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &transformResults, transform::TransformState &state) { … } void transform::PackOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } //===---------------------------------------------------------------------===// // PackGreedilyOp. //===---------------------------------------------------------------------===// LogicalResult transform::PackGreedilyOp::verify() { … } DiagnosedSilenceableFailure PackGreedilyOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &transformResults, transform::TransformState &state) { … } SmallVector<OpFoldResult> PackGreedilyOp::getMixedMatmulPackedSizes() { … } void transform::PackGreedilyOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } //===---------------------------------------------------------------------===// // PackTransposeOp //===---------------------------------------------------------------------===// LogicalResult transform::PackTransposeOp::verify() { … } namespace { enum class OuterOrInnerPerm { … }; } // namespace /// Return true if `permutation` is a valid permutation of the /// `outer_dims_perm` (case OuterOrInnerPerm::Outer) or `inner_dims_pos` /// (OuterOrInnerPerm::Inner) of the `tensor.pack` or `tensor.unpack` `op. /// This is the case when the `permutation` rank matches the rank expected by /// `op` and `permutation` is itself a permutation vector. /// Return true if either `op` or `permutation` are empty to allow a simpler /// polymorphic implementation. template <typename RelayoutOpTy> bool isValidPackingPermutation( RelayoutOpTy op, ArrayRef<int64_t> permutation, OuterOrInnerPerm outerOrInnerPerm = OuterOrInnerPerm::Outer) { … } DiagnosedSilenceableFailure transform::PackTransposeOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &transformResults, transform::TransformState &state) { … } //===---------------------------------------------------------------------===// // PadOp //===---------------------------------------------------------------------===// void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target, ArrayRef<int64_t> paddingDimensions, ArrayRef<int64_t> padToMultipleOf, ArrayRef<int64_t> packPaddings, ArrayRef<Attribute> transposePaddings, StringRef copyBackOp) { … } void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target, ArrayRef<int64_t> paddingDimensions, ArrayRef<OpFoldResult> mixedPadToMultipleOf, ArrayRef<int64_t> packPaddings, ArrayRef<Attribute> transposePaddings, StringRef copyBackOp) { … } void PadOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } SmallVector<OpFoldResult> PadOp::getMixedPadToMultipleOf() { … } DiagnosedSilenceableFailure transform::PadOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &results, transform::TransformState &state) { … } LogicalResult transform::PadOp::verify() { … } //===---------------------------------------------------------------------===// // HoistPadOp //===---------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::HoistPadBuildPackingLoopNestOp::apply( transform::TransformRewriter &rewriter, transform::TransformResults &transformResults, transform::TransformState &state) { … } LogicalResult transform::HoistPadBuildPackingLoopNestOp::verify() { … } void transform::HoistPadBuildPackingLoopNestOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } DiagnosedSilenceableFailure transform::HoistPadOp::applyToOne(transform::TransformRewriter &rewriter, tensor::PadOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } LogicalResult transform::HoistPadOp::verify() { … } //===----------------------------------------------------------------------===// // PromoteOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::PromoteOp::applyToOne(transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // ReplaceOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::ReplaceOp::apply(transform::TransformRewriter &rewriter, TransformResults &transformResults, TransformState &state) { … } void transform::ReplaceOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } LogicalResult transform::ReplaceOp::verify() { … } //===----------------------------------------------------------------------===// // ScalarizeOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::ScalarizeOp::applyToOne(transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // ConvertToLoopsOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::ConvertToLoopsOp::apply(transform::TransformRewriter &rewriter, transform::TransformResults &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // RewriteInDestinationPassingStyleOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::RewriteInDestinationPassingStyleOp::applyToOne( transform::TransformRewriter &rewriter, Operation *target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // SplitOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure SplitOp::apply(transform::TransformRewriter &rewriter, TransformResults &results, TransformState &state) { … } void SplitOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } ParseResult SplitOp::parse(OpAsmParser &parser, OperationState &result) { … } void SplitOp::print(OpAsmPrinter &printer) { … } LogicalResult SplitOp::verify() { … } //===----------------------------------------------------------------------===// // SplitReductionOp //===----------------------------------------------------------------------===// void transform::SplitReductionOp::build( OpBuilder &builder, OperationState &result, Value target, int64_t splitFactor, int64_t insertSplitDimension, bool innerParallel, bool useScalingAlgorithm, bool useAlloc) { … } DiagnosedSilenceableFailure transform::SplitReductionOp::applyToOne( transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // TileReductionUsingForOp //===----------------------------------------------------------------------===// void transform::TileReductionUsingForOp::build( OpBuilder &builder, OperationState &result, Value target, ArrayRef<int64_t> staticTileSizes) { … } DiagnosedSilenceableFailure transform::TileReductionUsingForOp::applyToOne( transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // TileReductionUsingForallOp //===----------------------------------------------------------------------===// void transform::TileReductionUsingForallOp::build( OpBuilder &builder, OperationState &result, Value target, ArrayRef<int64_t> staticNumThreads, ArrayRef<int64_t> staticTileSizes, ArrayAttr mapping) { … } DiagnosedSilenceableFailure transform::TileReductionUsingForallOp::applyToOne( transform::TransformRewriter &rewriter, LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // ContinuousTileSizesOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::ContinuousTileSizesOp::apply(transform::TransformRewriter &rewriter, TransformResults &transformResults, TransformState &state) { … } LogicalResult transform::ContinuousTileSizesOp::verify() { … } void transform::ContinuousTileSizesOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } static void printContinuousTileSizeTypes(OpAsmPrinter &printer, Operation *op, Type targetType, Type tile_sizes, Type) { … } static ParseResult parseContinuousTileSizeTypes(OpAsmParser &parser, Type &targetType, Type &tileSizesType, Type &chunkSizesType) { … } //===----------------------------------------------------------------------===// // TileUsingForOp //===----------------------------------------------------------------------===// void transform::TileUsingForOp::build( OpBuilder &builder, OperationState &result, TypeRange loopTypes, Value target, ArrayRef<int64_t> staticTileSizes, ArrayRef<int64_t> interchange, std::optional<ArrayRef<bool>> scalableSizes) { … } void transform::TileUsingForOp::build( OpBuilder &builder, OperationState &result, Value target, ArrayRef<int64_t> staticTileSizes, ArrayRef<int64_t> interchange, std::optional<ArrayRef<bool>> scalableSizes) { … } void transform::TileUsingForOp::build( OpBuilder &builder, OperationState &result, Value target, ArrayRef<OpFoldResult> mixedTileSizes, ArrayRef<int64_t> interchange, std::optional<ArrayRef<bool>> scalableSizes) { … } void transform::TileUsingForOp::build( OpBuilder &builder, OperationState &result, TypeRange loopTypes, Value target, ArrayRef<OpFoldResult> mixedTileSizes, ArrayRef<int64_t> interchange, std::optional<ArrayRef<bool>> scalableSizes) { … } LogicalResult transform::TileUsingForOp::verify() { … } DiagnosedSilenceableFailure transform::TileUsingForOp::apply(transform::TransformRewriter &rewriter, TransformResults &transformResults, TransformState &state) { … } SmallVector<OpFoldResult> transform::TileUsingForOp::getMixedSizes() { … } void transform::TileUsingForOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } //===----------------------------------------------------------------------===// // TileUsingForallOp //===----------------------------------------------------------------------===// void transform::TileUsingForallOp::build(OpBuilder &builder, OperationState &result, Value target, ArrayRef<int64_t> staticTileSizes, transform::TileSizesSpec, ArrayAttr mapping) { … } void transform::TileUsingForallOp::build(OpBuilder &builder, OperationState &result, Value target, ArrayRef<OpFoldResult> mixedTileSizes, transform::TileSizesSpec, ArrayAttr mapping) { … } void transform::TileUsingForallOp::build(OpBuilder &builder, OperationState &result, Value target, ArrayRef<int64_t> staticNumThreads, transform::NumThreadsSpec, ArrayAttr mapping) { … } void transform::TileUsingForallOp::build(OpBuilder &builder, OperationState &result, Value target, ArrayRef<OpFoldResult> mixedNumThreads, transform::NumThreadsSpec, ArrayAttr mapping) { … } /// Given `lbs`, `ubs` and `steps` of loops, return (for each loop), the /// normalized upper bound. static SmallVector<OpFoldResult> normalizeUpperBounds(RewriterBase &rewriter, Location loc, ArrayRef<OpFoldResult> lbs, ArrayRef<OpFoldResult> ubs, ArrayRef<OpFoldResult> steps) { … } /// When a loop is normalized, the uses of the induction variable within the /// loop need to replaced with `original_lb + old_iv * original_step`. static SmallVector<Value> denormalizeIndVar(RewriterBase &rewriter, Location loc, ValueRange ivs, ArrayRef<OpFoldResult> lbs, ArrayRef<OpFoldResult> steps) { … } /// Given a `scf.forall` loop return a loop op with the loop bounds /// normalized. /// TODO: Replace this with a general utility to normalize `scf.forall`. /// At the time of writing, this wasnt done since adding this to `scf` /// dialect would disallow using of `affine.apply` operations due /// to cyclic dependencies. To avoid churn in lit tests /// with the change this was added with, defer that to a follow up. static scf::ForallOp normalizeForallLoopOp(RewriterBase &rewriter, scf::ForallOp loop) { … } DiagnosedSilenceableFailure transform::tileToForallOpImpl( RewriterBase &rewriter, transform::TransformState &state, TransformOpInterface transformOp, Operation *target, ArrayRef<OpFoldResult> mixedNumThreads, ArrayRef<OpFoldResult> mixedTileSizes, std::optional<ArrayAttr> mapping, scf::SCFTilingResult &tilingResult) { … } DiagnosedSilenceableFailure transform::TileUsingForallOp::apply( transform::TransformRewriter &rewriter, transform::TransformResults &transformResults, transform::TransformState &state) { … } void transform::TileUsingForallOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } SmallVector<OpFoldResult> TileUsingForallOp::getMixedNumThreads() { … } SmallVector<OpFoldResult> TileUsingForallOp::getMixedTileSizes() { … } LogicalResult TileUsingForallOp::verify() { … } //===----------------------------------------------------------------------===// // VectorizeChildrenAndApplyPatternsOp //===----------------------------------------------------------------------===// void transform::VectorizeChildrenAndApplyPatternsOp::build( OpBuilder &builder, OperationState &result, Value target, bool vectorizePadding, bool vectorizeExtract, bool flatten1DDepthwiseConv) { … } namespace { /// This is an helper only to call vectorize via a pattern inside of /// VectorizeChildrenAndApplyPatternsOp::applyToOne. struct VectorizationPattern : public RewritePattern { … }; } // namespace DiagnosedSilenceableFailure transform::VectorizeChildrenAndApplyPatternsOp::applyToOne( transform::TransformRewriter &rewriter, Operation *target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // VectorizeOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::VectorizeOp::apply( transform::TransformRewriter &rewriter, mlir::transform::TransformResults &transformResults, mlir::transform::TransformState &state) { … } void transform::VectorizeOp::getEffects( SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { … } SmallVector<OpFoldResult> VectorizeOp::getMixedVectorSizes() { … } LogicalResult transform::VectorizeOp::verify() { … } //===----------------------------------------------------------------------===// // HoistRedundantVectorTransfersOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::HoistRedundantVectorTransfersOp::applyToOne( transform::TransformRewriter &rewriter, func::FuncOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // HoistRedundantVectorBroadcastsOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::HoistRedundantVectorBroadcastsOp::applyToOne( transform::TransformRewriter &rewriter, mlir::Operation *target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // ConvertConv2DToImg2ColOp. //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::ConvertConv2DToImg2ColOp::applyToOne( transform::TransformRewriter &rewriter, linalg::LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // FlattenElementwiseLinalgOp. //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne( transform::TransformRewriter &rewriter, linalg::LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // TransposeConv2DOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::TransposeConv2DOp::applyToOne( transform::TransformRewriter &rewriter, linalg::LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // TransposeMatmulOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::TransposeMatmulOp::applyToOne( transform::TransformRewriter &rewriter, linalg::LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // InsertSliceToCopyOp //===----------------------------------------------------------------------===// template <typename OpTy> DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } DiagnosedSilenceableFailure transform::InsertSliceToCopyOp::applyToOne( transform::TransformRewriter &rewriter, Operation *targetOp, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // MapCopyToThreadsOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::MapCopyToThreadsOp::applyToOne( transform::TransformRewriter &rewriter, Operation *target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } //===----------------------------------------------------------------------===// // WinogradConv2DOp //===----------------------------------------------------------------------===// DiagnosedSilenceableFailure transform::WinogradConv2DOp::applyToOne( transform::TransformRewriter &rewriter, linalg::LinalgOp target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } DiagnosedSilenceableFailure transform::DecomposeWinogradOp::applyToOne( transform::TransformRewriter &rewriter, Operation *target, transform::ApplyToEachResultList &results, transform::TransformState &state) { … } #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOpsEnums.cpp.inc" #define GET_OP_CLASSES #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp.inc"
Codebase Browser
llvm