#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/Interfaces/FoldInterfaces.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/STLForwardCompat.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include <cstddef>
#include <iterator>
#include <optional>
#include <variant>
#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"
usingnamespacemlir;
usingnamespacemlir::omp;
static ArrayAttr makeArrayAttr(MLIRContext *context,
llvm::ArrayRef<Attribute> attrs) { … }
static DenseBoolArrayAttr
makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef<bool> boolArray) { … }
namespace {
struct MemRefPointerLikeModel
: public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
MemRefType> { … };
struct LLVMPointerPointerLikeModel
: public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
LLVM::LLVMPointerType> { … };
}
void OpenMPDialect::initialize() { … }
static ParseResult parseAllocateAndAllocator(
OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocateVars,
SmallVectorImpl<Type> &allocateTypes,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocatorVars,
SmallVectorImpl<Type> &allocatorTypes) { … }
static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op,
OperandRange allocateVars,
TypeRange allocateTypes,
OperandRange allocatorVars,
TypeRange allocatorTypes) { … }
template <typename ClauseAttr>
static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr) { … }
template <typename ClauseAttr>
void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr) { … }
static ParseResult parseLinearClause(
OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
SmallVectorImpl<Type> &linearTypes,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars) { … }
static void printLinearClause(OpAsmPrinter &p, Operation *op,
ValueRange linearVars, TypeRange linearTypes,
ValueRange linearStepVars) { … }
static LogicalResult verifyNontemporalClause(Operation *op,
OperandRange nontemporalVars) { … }
static LogicalResult verifyAlignedClause(Operation *op,
std::optional<ArrayAttr> alignments,
OperandRange alignedVars) { … }
static ParseResult
parseAlignedClause(OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &alignedVars,
SmallVectorImpl<Type> &alignedTypes,
ArrayAttr &alignmentsAttr) { … }
static void printAlignedClause(OpAsmPrinter &p, Operation *op,
ValueRange alignedVars, TypeRange alignedTypes,
std::optional<ArrayAttr> alignments) { … }
static ParseResult
verifyScheduleModifiers(OpAsmParser &parser,
SmallVectorImpl<SmallString<12>> &modifiers) { … }
static ParseResult
parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr,
ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,
std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,
Type &chunkType) { … }
static void printScheduleClause(OpAsmPrinter &p, Operation *op,
ClauseScheduleKindAttr scheduleKind,
ScheduleModifierAttr scheduleMod,
UnitAttr scheduleSimd, Value scheduleChunk,
Type scheduleChunkType) { … }
static ParseResult parseOrderClause(OpAsmParser &parser,
ClauseOrderKindAttr &order,
OrderModifierAttr &orderMod) { … }
static void printOrderClause(OpAsmPrinter &p, Operation *op,
ClauseOrderKindAttr order,
OrderModifierAttr orderMod) { … }
namespace {
struct MapParseArgs { … };
struct PrivateParseArgs { … };
struct ReductionParseArgs { … };
struct AllRegionParseArgs { … };
}
static ParseResult parseClauseWithRegionArgs(
OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,
SmallVectorImpl<Type> &types,
SmallVectorImpl<OpAsmParser::Argument> ®ionPrivateArgs,
ArrayAttr *symbols = nullptr, DenseBoolArrayAttr *byref = nullptr) { … }
static ParseResult parseBlockArgClause(
OpAsmParser &parser,
llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
StringRef keyword, std::optional<MapParseArgs> mapArgs) { … }
static ParseResult parseBlockArgClause(
OpAsmParser &parser,
llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
StringRef keyword, std::optional<PrivateParseArgs> reductionArgs) { … }
static ParseResult parseBlockArgClause(
OpAsmParser &parser,
llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) { … }
static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region ®ion,
AllRegionParseArgs args) { … }
static ParseResult parseInReductionMapPrivateRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
SmallVectorImpl<Type> &inReductionTypes,
DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapVars,
SmallVectorImpl<Type> &mapTypes,
llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms) { … }
static ParseResult parseInReductionPrivateRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
SmallVectorImpl<Type> &inReductionTypes,
DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms) { … }
static ParseResult parseInReductionPrivateReductionRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
SmallVectorImpl<Type> &inReductionTypes,
DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
ArrayAttr &reductionSyms) { … }
static ParseResult parsePrivateRegion(
OpAsmParser &parser, Region ®ion,
llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms) { … }
static ParseResult parsePrivateReductionRegion(
OpAsmParser &parser, Region ®ion,
llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
ArrayAttr &reductionSyms) { … }
static ParseResult parseTaskReductionRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,
SmallVectorImpl<Type> &taskReductionTypes,
DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms) { … }
static ParseResult parseUseDeviceAddrUseDevicePtrRegion(
OpAsmParser &parser, Region ®ion,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDeviceAddrVars,
SmallVectorImpl<Type> &useDeviceAddrTypes,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDevicePtrVars,
SmallVectorImpl<Type> &useDevicePtrTypes) { … }
namespace {
struct MapPrintArgs { … };
struct PrivatePrintArgs { … };
struct ReductionPrintArgs { … };
struct AllRegionPrintArgs { … };
}
static void printClauseWithRegionArgs(OpAsmPrinter &p, MLIRContext *ctx,
StringRef clauseName,
ValueRange argsSubrange,
ValueRange operands, TypeRange types,
ArrayAttr symbols = nullptr,
DenseBoolArrayAttr byref = nullptr) { … }
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
StringRef clauseName, ValueRange argsSubrange,
std::optional<MapPrintArgs> mapArgs) { … }
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
StringRef clauseName, ValueRange argsSubrange,
std::optional<PrivatePrintArgs> privateArgs) { … }
static void
printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
ValueRange argsSubrange,
std::optional<ReductionPrintArgs> reductionArgs) { … }
static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region ®ion,
const AllRegionPrintArgs &args) { … }
static void printInReductionMapPrivateRegion(
OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange inReductionVars,
TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
ArrayAttr inReductionSyms, ValueRange mapVars, TypeRange mapTypes,
ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms) { … }
static void printInReductionPrivateRegion(
OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange inReductionVars,
TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
ArrayAttr privateSyms) { … }
static void printInReductionPrivateReductionRegion(
OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange inReductionVars,
TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
ArrayAttr privateSyms, ValueRange reductionVars, TypeRange reductionTypes,
DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) { … }
static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region ®ion,
ValueRange privateVars, TypeRange privateTypes,
ArrayAttr privateSyms) { … }
static void printPrivateReductionRegion(
OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange privateVars,
TypeRange privateTypes, ArrayAttr privateSyms, ValueRange reductionVars,
TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,
ArrayAttr reductionSyms) { … }
static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,
Region ®ion,
ValueRange taskReductionVars,
TypeRange taskReductionTypes,
DenseBoolArrayAttr taskReductionByref,
ArrayAttr taskReductionSyms) { … }
static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op,
Region ®ion,
ValueRange useDeviceAddrVars,
TypeRange useDeviceAddrTypes,
ValueRange useDevicePtrVars,
TypeRange useDevicePtrTypes) { … }
static LogicalResult
verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductionSyms,
OperandRange reductionVars,
std::optional<ArrayRef<bool>> reductionByref) { … }
static ParseResult parseCopyprivate(
OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> ©privateVars,
SmallVectorImpl<Type> ©privateTypes, ArrayAttr ©privateSyms) { … }
static void printCopyprivate(OpAsmPrinter &p, Operation *op,
OperandRange copyprivateVars,
TypeRange copyprivateTypes,
std::optional<ArrayAttr> copyprivateSyms) { … }
static LogicalResult
verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars,
std::optional<ArrayAttr> copyprivateSyms) { … }
static ParseResult
parseDependVarList(OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dependVars,
SmallVectorImpl<Type> &dependTypes, ArrayAttr &dependKinds) { … }
static void printDependVarList(OpAsmPrinter &p, Operation *op,
OperandRange dependVars, TypeRange dependTypes,
std::optional<ArrayAttr> dependKinds) { … }
static LogicalResult verifyDependVarList(Operation *op,
std::optional<ArrayAttr> dependKinds,
OperandRange dependVars) { … }
static ParseResult parseSynchronizationHint(OpAsmParser &parser,
IntegerAttr &hintAttr) { … }
static void printSynchronizationHint(OpAsmPrinter &p, Operation *op,
IntegerAttr hintAttr) { … }
static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint) { … }
uint64_t mapTypeToBitFlag(uint64_t value,
llvm::omp::OpenMPOffloadMappingFlags flag) { … }
static ParseResult parseMapClause(OpAsmParser &parser, IntegerAttr &mapType) { … }
static void printMapClause(OpAsmPrinter &p, Operation *op,
IntegerAttr mapType) { … }
static ParseResult parseMembersIndex(OpAsmParser &parser,
DenseIntElementsAttr &membersIdx) { … }
static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,
DenseIntElementsAttr membersIdx) { … }
static void printCaptureType(OpAsmPrinter &p, Operation *op,
VariableCaptureKindAttr mapCaptureType) { … }
static ParseResult parseCaptureType(OpAsmParser &parser,
VariableCaptureKindAttr &mapCaptureType) { … }
static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars) { … }
void TargetDataOp::build(OpBuilder &builder, OperationState &state,
const TargetDataOperands &clauses) { … }
LogicalResult TargetDataOp::verify() { … }
void TargetEnterDataOp::build(
OpBuilder &builder, OperationState &state,
const TargetEnterExitUpdateDataOperands &clauses) { … }
LogicalResult TargetEnterDataOp::verify() { … }
void TargetExitDataOp::build(OpBuilder &builder, OperationState &state,
const TargetEnterExitUpdateDataOperands &clauses) { … }
LogicalResult TargetExitDataOp::verify() { … }
void TargetUpdateOp::build(OpBuilder &builder, OperationState &state,
const TargetEnterExitUpdateDataOperands &clauses) { … }
LogicalResult TargetUpdateOp::verify() { … }
void TargetOp::build(OpBuilder &builder, OperationState &state,
const TargetOperands &clauses) { … }
LogicalResult TargetOp::verify() { … }
void ParallelOp::build(OpBuilder &builder, OperationState &state,
ArrayRef<NamedAttribute> attributes) { … }
void ParallelOp::build(OpBuilder &builder, OperationState &state,
const ParallelOperands &clauses) { … }
template <typename OpType>
static LogicalResult verifyPrivateVarList(OpType &op) { … }
LogicalResult ParallelOp::verify() { … }
static bool opInGlobalImplicitParallelRegion(Operation *op) { … }
void TeamsOp::build(OpBuilder &builder, OperationState &state,
const TeamsOperands &clauses) { … }
LogicalResult TeamsOp::verify() { … }
unsigned SectionOp::numPrivateBlockArgs() { … }
unsigned SectionOp::numReductionBlockArgs() { … }
void SectionsOp::build(OpBuilder &builder, OperationState &state,
const SectionsOperands &clauses) { … }
LogicalResult SectionsOp::verify() { … }
LogicalResult SectionsOp::verifyRegions() { … }
void SingleOp::build(OpBuilder &builder, OperationState &state,
const SingleOperands &clauses) { … }
LogicalResult SingleOp::verify() { … }
LogicalResult LoopWrapperInterface::verifyImpl() { … }
void WsloopOp::build(OpBuilder &builder, OperationState &state,
ArrayRef<NamedAttribute> attributes) { … }
void WsloopOp::build(OpBuilder &builder, OperationState &state,
const WsloopOperands &clauses) { … }
LogicalResult WsloopOp::verify() { … }
void SimdOp::build(OpBuilder &builder, OperationState &state,
const SimdOperands &clauses) { … }
LogicalResult SimdOp::verify() { … }
void DistributeOp::build(OpBuilder &builder, OperationState &state,
const DistributeOperands &clauses) { … }
LogicalResult DistributeOp::verify() { … }
LogicalResult DeclareReductionOp::verifyRegions() { … }
void TaskOp::build(OpBuilder &builder, OperationState &state,
const TaskOperands &clauses) { … }
LogicalResult TaskOp::verify() { … }
void TaskgroupOp::build(OpBuilder &builder, OperationState &state,
const TaskgroupOperands &clauses) { … }
LogicalResult TaskgroupOp::verify() { … }
void TaskloopOp::build(OpBuilder &builder, OperationState &state,
const TaskloopOperands &clauses) { … }
SmallVector<Value> TaskloopOp::getAllReductionVars() { … }
LogicalResult TaskloopOp::verify() { … }
ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) { … }
void LoopNestOp::print(OpAsmPrinter &p) { … }
void LoopNestOp::build(OpBuilder &builder, OperationState &state,
const LoopNestOperands &clauses) { … }
LogicalResult LoopNestOp::verify() { … }
void LoopNestOp::gatherWrappers(
SmallVectorImpl<LoopWrapperInterface> &wrappers) { … }
void CriticalDeclareOp::build(OpBuilder &builder, OperationState &state,
const CriticalDeclareOperands &clauses) { … }
LogicalResult CriticalDeclareOp::verify() { … }
LogicalResult CriticalOp::verifySymbolUses(SymbolTableCollection &symbolTable) { … }
static LogicalResult verifyOrderedParent(Operation &op) { … }
void OrderedOp::build(OpBuilder &builder, OperationState &state,
const OrderedOperands &clauses) { … }
LogicalResult OrderedOp::verify() { … }
void OrderedRegionOp::build(OpBuilder &builder, OperationState &state,
const OrderedRegionOperands &clauses) { … }
LogicalResult OrderedRegionOp::verify() { … }
void TaskwaitOp::build(OpBuilder &builder, OperationState &state,
const TaskwaitOperands &clauses) { … }
LogicalResult AtomicReadOp::verify() { … }
LogicalResult AtomicWriteOp::verify() { … }
LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
PatternRewriter &rewriter) { … }
LogicalResult AtomicUpdateOp::verify() { … }
LogicalResult AtomicUpdateOp::verifyRegions() { … }
AtomicReadOp AtomicCaptureOp::getAtomicReadOp() { … }
AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() { … }
AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() { … }
LogicalResult AtomicCaptureOp::verify() { … }
LogicalResult AtomicCaptureOp::verifyRegions() { … }
void CancelOp::build(OpBuilder &builder, OperationState &state,
const CancelOperands &clauses) { … }
LogicalResult CancelOp::verify() { … }
void CancellationPointOp::build(OpBuilder &builder, OperationState &state,
const CancellationPointOperands &clauses) { … }
LogicalResult CancellationPointOp::verify() { … }
LogicalResult MapBoundsOp::verify() { … }
void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
TypeRange , StringAttr symName,
TypeAttr type) { … }
LogicalResult PrivateClauseOp::verify() { … }
void MaskedOp::build(OpBuilder &builder, OperationState &state,
const MaskedOperands &clauses) { … }
#define GET_ATTRDEF_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
#define GET_OP_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
#define GET_TYPEDEF_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"