//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This provides a class for OpenMP runtime code generation. // //===----------------------------------------------------------------------===// #include "CGOpenMPRuntime.h" #include "ABIInfoImpl.h" #include "CGCXXABI.h" #include "CGCleanup.h" #include "CGRecordLayout.h" #include "CodeGenFunction.h" #include "TargetInfo.h" #include "clang/AST/APValue.h" #include "clang/AST/Attr.h" #include "clang/AST/Decl.h" #include "clang/AST/OpenMPClause.h" #include "clang/AST/StmtOpenMP.h" #include "clang/AST/StmtVisitor.h" #include "clang/Basic/BitmaskEnum.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/SourceManager.h" #include "clang/CodeGen/ConstantInitBuilder.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Bitcode/BitcodeReader.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Value.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include <cassert> #include <cstdint> #include <numeric> #include <optional> usingnamespaceclang; usingnamespaceCodeGen; usingnamespacellvm::omp; namespace { /// Base class for handling code generation inside OpenMP regions. class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { … }; /// API for captured statement code generation in OpenMP constructs. class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo { … }; /// API for captured statement code generation in OpenMP constructs. class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo { … }; /// API for inlined captured statement code generation in OpenMP /// constructs. class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo { … }; /// API for captured statement code generation in OpenMP target /// constructs. For this captures, implicit parameters are used instead of the /// captured fields. The name of the target region has to be unique in a given /// application so it is provided by the client, because only the client has /// the information to generate that. class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo { … }; static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) { … } /// API for generation of expressions captured in a innermost OpenMP /// region. class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo { … }; /// RAII for emitting code of OpenMP constructs. class InlinedOpenMPRegionRAII { … }; /// Values for bit flags used in the ident_t to describe the fields. /// All enumeric elements are named and described in accordance with the code /// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h enum OpenMPLocationFlags : unsigned { … }; /// Describes ident structure that describes a source location. /// All descriptions are taken from /// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h /// Original structure: /// typedef struct ident { /// kmp_int32 reserved_1; /**< might be used in Fortran; /// see above */ /// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; /// KMP_IDENT_KMPC identifies this union /// member */ /// kmp_int32 reserved_2; /**< not really used in Fortran any more; /// see above */ ///#if USE_ITT_BUILD /// /* but currently used for storing /// region-specific ITT */ /// /* contextual information. */ ///#endif /* USE_ITT_BUILD */ /// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for /// C++ */ /// char const *psource; /**< String describing the source location. /// The string is composed of semi-colon separated // fields which describe the source file, /// the function and a pair of line numbers that /// delimit the construct. /// */ /// } ident_t; enum IdentFieldIndex { … }; /// Schedule types for 'omp for' loops (these enumerators are taken from /// the enum sched_type in kmp.h). enum OpenMPSchedType { … }; /// A basic class for pre|post-action for advanced codegen sequence for OpenMP /// region. class CleanupTy final : public EHScopeStack::Cleanup { … }; } // anonymous namespace void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { … } /// Check if the combiner is a call to UDR combiner and if it is so return the /// UDR decl used for reduction. static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp) { … } static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty) { … } /// Emit initialization of arrays of complex types. /// \param DestAddr Address of the array. /// \param Type Type of array. /// \param Init Initial expression of array. /// \param SrcAddr Address of the original array. static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, bool EmitDeclareReductionInit, const Expr *Init, const OMPDeclareReductionDecl *DRD, Address SrcAddr = Address::invalid()) { … } LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { … } LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E) { … } void ReductionCodeGen::emitAggregateInitialization( CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, const OMPDeclareReductionDecl *DRD) { … } ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs, ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> ReductionOps) { … } void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { … } void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { … } void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size) { … } void ReductionCodeGen::emitInitialization( CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) { … } bool ReductionCodeGen::needCleanups(unsigned N) { … } void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr) { … } static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV) { … } static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, Address OriginalBaseAddress, llvm::Value *Addr) { … } static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) { … } Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, Address PrivateAddr) { … } bool ReductionCodeGen::usesReductionInitializer(unsigned N) const { … } LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { … } void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { … } LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue( CodeGenFunction &CGF) { … } static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy) { … } CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) : … { … } void CGOpenMPRuntime::clear() { … } std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const { … } static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner) { … } void CGOpenMPRuntime::emitUserDefinedReduction( CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { … } std::pair<llvm::Function *, llvm::Function *> CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) { … } namespace { // Temporary RAII solution to perform a push/pop stack event on the OpenMP IR // Builder if one is present. struct PushAndPopStackRAII { … }; } // namespace static llvm::Function *emitParallelOrTeamsOutlinedFunction( CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) { … } std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const { … } std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const { … } std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const { … } llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction( CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { … } llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction( CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { … } llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction( const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) { … } void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, bool AtCurrentPoint) { … } void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { … } static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, SourceLocation Loc, SmallString<128> &Buffer) { … } llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags, bool EmitLoc) { … } llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, SourceLocation Loc) { … } void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { … } llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() { … } llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() { … } static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind convertDeviceClause(const VarDecl *VD) { … } static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind convertCaptureClause(const VarDecl *VD) { … } static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc( CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, SourceLocation BeginLoc, llvm::StringRef ParentName = "") { … } ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { … } llvm::Constant * CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { … } Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) { … } void CGOpenMPRuntime::emitThreadPrivateVarInit( CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { … } llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF) { … } void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD, llvm::GlobalValue *GV) { … } Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, QualType VarType, StringRef Name) { … } void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen) { … } void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond, llvm::Value *NumThreads) { … } // If we're inside an (outlined) parallel region, use the region info's // thread-ID variable (it is passed in a first argument of the outlined function // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in // regular serial code region, get thread ID by calling kmp_int32 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and // return the address of that temp. Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc) { … } llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { … } namespace { /// Common pre(post)-action for different OpenMP constructs. class CommonActionTy final : public PrePostActionTy { … }; } // anonymous namespace void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint) { … } void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) { … } void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc, const Expr *Filter) { … } void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) { … } void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) { … } /// Given an array of pointers to variables, project the address of a /// given variable. static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var) { … } static llvm::Value *emitCopyprivateCopyFunction( CodeGenModule &CGM, llvm::Type *ArgsElemType, ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps, SourceLocation Loc) { … } void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> DstExprs, ArrayRef<const Expr *> AssignmentOps) { … } void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) { … } unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) { … } void CGOpenMPRuntime::getDefaultScheduleAndChunk( CodeGenFunction &CGF, const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const { … } void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks, bool ForceSimpleCall) { … } void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME, bool IsFatal) { … } /// Map the OpenMP loop schedule to the runtime enumeration. static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered) { … } /// Map the OpenMP distribute schedule to the runtime enumeration. static OpenMPSchedType getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) { … } bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const { … } bool CGOpenMPRuntime::isStaticNonchunked( OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const { … } bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const { … } bool CGOpenMPRuntime::isStaticChunked( OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const { … } bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const { … } static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2) { … } void CGOpenMPRuntime::emitForDispatchInit( CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) { … } void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) { … } static void emitForStaticInitCall( CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, const CGOpenMPRuntime::StaticRTInput &Values) { … } void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) { … } void CGOpenMPRuntime::emitDistributeStaticInit( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const CGOpenMPRuntime::StaticRTInput &Values) { … } void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) { … } void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) { … } llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) { … } void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) { … } void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, ProcBindKind ProcBind, SourceLocation Loc) { … } void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, SourceLocation Loc, llvm::AtomicOrdering AO) { … } namespace { /// Indexes of fields for type kmp_task_t. enum KmpTaskTFields { … }; } // anonymous namespace void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() { … } void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { … } namespace { struct PrivateHelpersTy { … }; PrivateDataTy; } // anonymous namespace static bool isAllocatableDecl(const VarDecl *VD) { … } static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) { … } static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy) { … } static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef<PrivateDataTy> Privates) { … } /// Emit a proxy function which accepts kmp_task_t as the second /// argument. /// \code /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { /// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt, /// For taskloops: /// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter, /// tt->reductions, tt->shareds); /// return 0; /// } /// \endcode static llvm::Function * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Function *TaskFunction, llvm::Value *TaskPrivatesMap) { … } static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy) { … } /// Emit a privates mapping function for correct handling of private and /// firstprivate variables. /// \code /// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1> /// **noalias priv1,..., <tyn> **noalias privn) { /// *priv1 = &.privates.priv1; /// ...; /// *privn = &.privates.privn; /// } /// \endcode static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPTaskDataTy &Data, QualType PrivatesQTy, ArrayRef<PrivateDataTy> Privates) { … } /// Emit initialization for private variables in task-based directives. static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef<PrivateDataTy> Privates, bool ForDup) { … } /// Check if duplication function is required for taskloops. static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef<PrivateDataTy> Privates) { … } /// Emit task_dup function (for initialization of /// private/firstprivate/lastprivate vars and last_iter flag) /// \code /// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int /// lastpriv) { /// // setup lastprivate flag /// task_dst->last = lastpriv; /// // could be constructor calls here... /// } /// \endcode static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef<PrivateDataTy> Privates, bool WithLastIter) { … } /// Checks if destructor function is required to be generated. /// \return true if cleanups are required, false otherwise. static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, ArrayRef<PrivateDataTy> Privates) { … } namespace { /// Loop generator for OpenMP iterator expression. class OMPIteratorGeneratorScope final : public CodeGenFunction::OMPPrivateScope { … }; } // namespace static std::pair<llvm::Value *, llvm::Value *> getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { … } /// Builds kmp_depend_info, if it is not built yet, and builds flags type. static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) { … } CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data) { … } /// Translates internal dependency kind into the runtime kind. static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) { … } /// Builds kmp_depend_info, if it is not built yet, and builds flags type. static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, QualType &FlagsTy) { … } std::pair<llvm::Value *, LValue> CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc) { … } static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, llvm::PointerUnion<unsigned *, LValue *> Pos, const OMPTaskDataTy::DependData &Data, Address DependenciesArray) { … } SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes( CodeGenFunction &CGF, QualType &KmpDependInfoTy, const OMPTaskDataTy::DependData &Data) { … } void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy, LValue PosLVal, const OMPTaskDataTy::DependData &Data, Address DependenciesArray) { … } std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause( CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, SourceLocation Loc) { … } Address CGOpenMPRuntime::emitDepobjDependClause( CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, SourceLocation Loc) { … } void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, SourceLocation Loc) { … } void CGOpenMPRuntime::emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, SourceLocation Loc) { … } void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) { … } void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) { … } /// Emit reduction operation for each element of array (required for /// array sections) LHS op = RHS. /// \param Type Type of array. /// \param LHSVar Variable on the left side of the reduction operation /// (references element of array in original variable). /// \param RHSVar Variable on the right side of the reduction operation /// (references element of array in original variable). /// \param RedOpGen Generator of reduction operation with use of LHSVar and /// RHSVar. static void EmitOMPAggregateReduction( CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr = nullptr, const Expr *EExpr = nullptr, const Expr *UpExpr = nullptr) { … } /// Emit reduction combiner. If the combiner is a simple expression emit it as /// is, otherwise consider it as combiner of UDR decl and emit it as a call of /// UDR combiner function. static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp) { … } llvm::Function *CGOpenMPRuntime::emitReductionFunction( StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) { … } void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS) { … } void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) { … } /// Generates unique name for artificial threadprivate variables. /// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>" static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, const Expr *Ref) { … } /// Emits reduction initializer function: /// \code /// void @.red_init(void* %arg, void* %orig) { /// %0 = bitcast void* %arg to <type>* /// store <type> <init>, <type>* %0 /// ret void /// } /// \endcode static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) { … } /// Emits reduction combiner function: /// \code /// void @.red_comb(void* %arg0, void* %arg1) { /// %lhs = bitcast void* %arg0 to <type>* /// %rhs = bitcast void* %arg1 to <type>* /// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs) /// store <type> %2, <type>* %lhs /// ret void /// } /// \endcode static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N, const Expr *ReductionOp, const Expr *LHS, const Expr *RHS, const Expr *PrivateRef) { … } /// Emits reduction finalizer function: /// \code /// void @.red_fini(void* %arg) { /// %0 = bitcast void* %arg to <type>* /// <destroy>(<type>* %0) /// ret void /// } /// \endcode static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) { … } llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) { … } void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) { … } void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) { … } Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) { … } void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) { … } void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnerKind, const RegionCodeGenTy &CodeGen, bool HasCancel) { … } namespace { enum RTCancelKind { … }; } // anonymous namespace static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) { … } void CGOpenMPRuntime::emitCancellationPointCall( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) { … } void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) { … } namespace { /// Cleanup action for uses_allocators support. class OMPUsesAllocatorsActionTy final : public PrePostActionTy { … }; } // namespace void CGOpenMPRuntime::emitTargetOutlinedFunction( const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { … } void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator, const Expr *AllocatorTraits) { … } void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator) { … } void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams( const OMPExecutableDirective &D, CodeGenFunction &CGF, int32_t &MinThreadsVal, int32_t &MaxThreadsVal, int32_t &MinTeamsVal, int32_t &MaxTeamsVal) { … } void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper( const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { … } /// Checks if the expression is constant or does not have non-trivial function /// calls. static bool isTrivial(ASTContext &Ctx, const Expr * E) { … } const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body) { … } const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective( CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, int32_t &MaxTeamsVal) { … } llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective( CodeGenFunction &CGF, const OMPExecutableDirective &D) { … } /// Check for a num threads constant value (stored in \p DefaultVal), or /// expression (stored in \p E). If the value is conditional (via an if-clause), /// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are /// nullptr, no expression evaluation is perfomed. static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, const Expr **E, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal) { … } const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective( CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) { … } llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective( CodeGenFunction &CGF, const OMPExecutableDirective &D) { … } namespace { LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(…); // Utility to handle information from clauses associated with a given // construct that use mappable expressions (e.g. 'map' clause, 'to' clause). // It provides a convenient interface to obtain the information and generate // code for that information. class MappableExprsHandler { … }; } // anonymous namespace // Try to extract the base declaration from a `this->x` expression if possible. static ValueDecl *getDeclFromThisExpr(const Expr *E) { … } /// Emit a string constant containing the names of the values mapped to the /// offloading runtime library. static llvm::Constant * emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MappingExprInfo &MapExprs) { … } /// Emit the arrays used to pass the captures and map information to the /// offloading runtime library. If there is no map or capture information, /// return nullptr by reference. static void emitOffloadingArraysAndArgs( CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, bool IsNonContiguous = false, bool ForEndCall = false) { … } /// Check for inner distribute directive. static const OMPExecutableDirective * getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) { … } /// Emit the user-defined mapper function. The code generation follows the /// pattern in the example below. /// \code /// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle, /// void *base, void *begin, /// int64_t size, int64_t type, /// void *name = nullptr) { /// // Allocate space for an array section first or add a base/begin for /// // pointer dereference. /// if ((size > 1 || (base != begin && maptype.IsPtrAndObj)) && /// !maptype.IsDelete) /// __tgt_push_mapper_component(rt_mapper_handle, base, begin, /// size*sizeof(Ty), clearToFromMember(type)); /// // Map members. /// for (unsigned i = 0; i < size; i++) { /// // For each component specified by this mapper: /// for (auto c : begin[i]->all_components) { /// if (c.hasMapper()) /// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size, /// c.arg_type, c.arg_name); /// else /// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base, /// c.arg_begin, c.arg_size, c.arg_type, /// c.arg_name); /// } /// } /// // Delete the array section. /// if (size > 1 && maptype.IsDelete) /// __tgt_push_mapper_component(rt_mapper_handle, base, begin, /// size*sizeof(Ty), clearToFromMember(type)); /// } /// \endcode void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D, CodeGenFunction *CGF) { … } /// Emit the array initialization or deletion portion for user-defined mapper /// code generation. First, it evaluates whether an array section is mapped and /// whether the \a MapType instructs to delete this section. If \a IsInit is /// true, and \a MapType indicates to not delete this array, array /// initialization code is generated. If \a IsInit is false, and \a MapType /// indicates to not this array, array deletion code is generated. void CGOpenMPRuntime::emitUDMapperArrayInitOrDel( CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *Base, llvm::Value *Begin, llvm::Value *Size, llvm::Value *MapType, llvm::Value *MapName, CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit) { … } llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc( const OMPDeclareMapperDecl *D) { … } llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall( CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) { … } static void emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF) { … } static llvm::Value *emitDeviceID( llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, CodeGenFunction &CGF) { … } static llvm::Value *emitDynCGGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { … } static void genMapInfoForCaptures( MappableExprsHandler &MEHandler, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, llvm::OpenMPIRBuilder &OMPBuilder, llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) { … } static void genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder, const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet = llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) { … } static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF, const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, llvm::OpenMPIRBuilder &OMPBuilder, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) { … } template <typename ClauseTy> static void emitClauseForBareTargetDirective(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::SmallVectorImpl<llvm::Value *> &Values) { … } static void emitTargetCallKernelLaunch( CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter, CodeGenFunction &CGF, CodeGenModule &CGM) { … } static void emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, const OMPExecutableDirective &D, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask, const CapturedStmt &CS, bool OffloadingMandatory, CodeGenFunction &CGF) { … } void CGOpenMPRuntime::emitTargetCall( CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) { … } void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName) { … } static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) { … } bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) { … } bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) { … } void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD, llvm::Constant *Addr) { … } bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) { … } void CGOpenMPRuntime::emitDeferredTargetDecls() const { … } void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas( CodeGenFunction &CGF, const OMPExecutableDirective &D) const { … } void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) { … } llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const { … } bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) { … } bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const { … } CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII( CodeGenModule &CGM) : … { … } CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() { … } bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) { … } void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef<llvm::Value *> CapturedVars) { … } void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) { … } void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF, const Expr *ThreadLimit, SourceLocation Loc) { … } void CGOpenMPRuntime::emitTargetDataCalls( CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) { … } void CGOpenMPRuntime::emitTargetDataStandAloneCall( CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) { … } namespace { /// Kind of parameter in a function with 'declare simd' directive. enum ParamKindTy { … }; /// Attribute set of the parameter. struct ParamAttrTy { … }; } // namespace static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) { … } /// Mangle the parameter part of the vector function name according to /// their OpenMP classification. The mangling function is defined in /// section 4.5 of the AAVFABI(2021Q1). static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) { … } static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, const llvm::APSInt &VLENVal, ArrayRef<ParamAttrTy> ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State) { … } // This are the Functions that are needed to mangle the name of the // vector functions generated by the compiler, according to the rules // defined in the "Vector Function ABI specifications for AArch64", // available at // https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi. /// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1). static bool getAArch64MTV(QualType QT, ParamKindTy Kind) { … } /// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI. static bool getAArch64PBV(QualType QT, ASTContext &C) { … } /// Computes the lane size (LS) of a return type or of an input parameter, /// as defined by `LS(P)` in 3.2.1 of the AAVFABI. /// TODO: Add support for references, section 3.2.1, item 1. static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) { … } // Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the // signature of the scalar function, as defined in 3.2.2 of the // AAVFABI. static std::tuple<unsigned, unsigned, bool> getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) { … } // Function used to add the attribute. The parameter `VLEN` is // templated to allow the use of "x" when targeting scalable functions // for SVE. template <typename T> static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn) { … } // Helper function to generate the Advanced SIMD names depending on // the value of the NDS when simdlen is not present. static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, StringRef Prefix, char ISA, StringRef ParSeq, StringRef MangledName, bool OutputBecomesInput, llvm::Function *Fn) { … } /// Emit vector function attributes for AArch64, as defined in the AAVFABI. static void emitAArch64DeclareSimdFunction( CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, ArrayRef<ParamAttrTy> ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) { … } void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn) { … } namespace { /// Cleanup action for doacross support. class DoacrossCleanupTy final : public EHScopeStack::Cleanup { … }; } // namespace void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef<Expr *> NumIterations) { … } template <typename T> static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, const T *C, llvm::Value *ULoc, llvm::Value *ThreadID) { … } void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) { … } void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDoacrossClause *C) { … } void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee Callee, ArrayRef<llvm::Value *> Args) const { … } void CGOpenMPRuntime::emitOutlinedFunctionCall( CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, ArrayRef<llvm::Value *> Args) const { … } void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { … } Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const { … } /// Return allocator value from expression, or return a null allocator (default /// when no allocator specified). static llvm::Value *getAllocatorVal(CodeGenFunction &CGF, const Expr *Allocator) { … } /// Return the alignment from an allocate directive if present. static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) { … } Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, const VarDecl *VD) { … } bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const { … } CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII( CodeGenModule &CGM, const OMPLoopDirective &S) : … { … } CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() { … } CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII( CodeGenFunction &CGF, const llvm::MapVector<CanonicalDeclPtr<const VarDecl>, std::pair<Address, Address>> &LocalVars) : … { … } CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() { … } bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const { … } void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis( const OMPExecutableDirective &S, llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled) const { … } CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII( CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) : … { … } CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII( CodeGenFunction &CGF, const OMPExecutableDirective &S) : … { … } CGOpenMPRuntime::LastprivateConditionalRAII CGOpenMPRuntime::LastprivateConditionalRAII::disable( CodeGenFunction &CGF, const OMPExecutableDirective &S) { … } CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() { … } Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, const VarDecl *VD) { … } namespace { /// Checks if the lastprivate conditional variable is referenced in LHS. class LastprivateConditionalRefChecker final : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> { … }; } // namespace void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal, StringRef UniqueDeclName, LValue LVal, SourceLocation Loc) { … } void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, const Expr *LHS) { … } void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional( CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) { … } void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, SourceLocation Loc) { … } llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction( CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { … } llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction( CodeGenFunction &CGF, const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { … } llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction( const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts) { … } void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond, llvm::Value *NumThreads) { … } void CGOpenMPSIMDRuntime::emitCriticalRegion( CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint) { … } void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) { … } void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc, const Expr *Filter) { … } void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) { … } void CGOpenMPSIMDRuntime::emitTaskgroupRegion( CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc) { … } void CGOpenMPSIMDRuntime::emitSingleRegion( CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) { … } void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads) { … } void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks, bool ForceSimpleCall) { … } void CGOpenMPSIMDRuntime::emitForDispatchInit( CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, const DispatchRTInput &DispatchValues) { … } void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) { … } void CGOpenMPSIMDRuntime::emitForStaticInit( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) { … } void CGOpenMPSIMDRuntime::emitDistributeStaticInit( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) { … } void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned) { … } void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind) { … } llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST) { … } void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc) { … } void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, ProcBindKind ProcBind, SourceLocation Loc) { … } Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc) { … } llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition( const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF) { … } Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate( CodeGenFunction &CGF, QualType VarType, StringRef Name) { … } void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars, SourceLocation Loc, llvm::AtomicOrdering AO) { … } void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) { … } void CGOpenMPSIMDRuntime::emitTaskLoopCall( CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data) { … } void CGOpenMPSIMDRuntime::emitReduction( CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) { … } llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit( CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) { … } void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc, bool IsWorksharingReduction) { … } void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc, ReductionCodeGen &RCG, unsigned N) { … } Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *ReductionsPtr, LValue SharedLVal) { … } void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPTaskDataTy &Data) { … } void CGOpenMPSIMDRuntime::emitCancellationPointCall( CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion) { … } void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion) { … } void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction( const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { … } void CGOpenMPSIMDRuntime::emitTargetCall( CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, const OMPLoopDirective &D)> SizeEmitter) { … } bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) { … } bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) { … } bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) { … } void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Function *OutlinedFn, ArrayRef<llvm::Value *> CapturedVars) { … } void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc) { … } void CGOpenMPSIMDRuntime::emitTargetDataCalls( CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen, CGOpenMPRuntime::TargetDataInfo &Info) { … } void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall( CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device) { … } void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D, ArrayRef<Expr *> NumIterations) { … } void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C) { … } void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDoacrossClause *C) { … } const VarDecl * CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD, const VarDecl *NativeParam) const { … } Address CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, const VarDecl *TargetParam) const { … }
Codebase Browser
llvm