/* * kmp_dispatch.cpp: dynamic scheduling - iteration initialization and dispatch. */ //===----------------------------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /* Dynamic scheduling initialization and dispatch. * * NOTE: __kmp_nth is a constant inside of any dispatch loop, however * it may change values between parallel regions. __kmp_max_nth * is the largest value __kmp_nth may take, 1 is the smallest. */ #include "kmp.h" #include "kmp_error.h" #include "kmp_i18n.h" #include "kmp_itt.h" #include "kmp_stats.h" #include "kmp_str.h" #if KMP_USE_X87CONTROL #include <float.h> #endif #include "kmp_lock.h" #include "kmp_dispatch.h" #if KMP_USE_HIER_SCHED #include "kmp_dispatch_hier.h" #endif #if OMPT_SUPPORT #include "ompt-specific.h" #endif /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ void __kmp_dispatch_deo_error(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { … } void __kmp_dispatch_dxo_error(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { … } // Returns either SCHEDULE_MONOTONIC or SCHEDULE_NONMONOTONIC static inline int __kmp_get_monotonicity(ident_t *loc, enum sched_type schedule, bool use_hier = false) { … } #if KMP_WEIGHTED_ITERATIONS_SUPPORTED // Return floating point number rounded to two decimal points static inline float __kmp_round_2decimal_val(float num) { … } static inline int __kmp_get_round_val(float num) { … } #endif template <typename T> inline void __kmp_initialize_self_buffer(kmp_team_t *team, T id, dispatch_private_info_template<T> *pr, typename traits_t<T>::unsigned_t nchunks, T nproc, typename traits_t<T>::unsigned_t &init, T &small_chunk, T &extras, T &p_extra) { … } #if KMP_STATIC_STEAL_ENABLED enum { … }; #endif // Initialize a dispatch_private_info_template<T> buffer for a particular // type of schedule,chunk. The loop description is found in lb (lower bound), // ub (upper bound), and st (stride). nproc is the number of threads relevant // to the scheduling (often the number of threads in a team, but not always if // hierarchical scheduling is used). tid is the id of the thread calling // the function within the group of nproc threads. It will have a value // between 0 and nproc - 1. This is often just the thread id within a team, but // is not necessarily the case when using hierarchical scheduling. // loc is the source file location of the corresponding loop // gtid is the global thread id template <typename T> void __kmp_dispatch_init_algorithm(ident_t *loc, int gtid, dispatch_private_info_template<T> *pr, enum sched_type schedule, T lb, T ub, typename traits_t<T>::signed_t st, #if USE_ITT_BUILD kmp_uint64 *cur_chunk, #endif typename traits_t<T>::signed_t chunk, T nproc, T tid) { … } #if KMP_USE_HIER_SCHED template <typename T> inline void __kmp_dispatch_init_hier_runtime(ident_t *loc, T lb, T ub, typename traits_t<T>::signed_t st); template <> inline void __kmp_dispatch_init_hier_runtime<kmp_int32>(ident_t *loc, kmp_int32 lb, kmp_int32 ub, kmp_int32 st) { __kmp_dispatch_init_hierarchy<kmp_int32>( loc, __kmp_hier_scheds.size, __kmp_hier_scheds.layers, __kmp_hier_scheds.scheds, __kmp_hier_scheds.small_chunks, lb, ub, st); } template <> inline void __kmp_dispatch_init_hier_runtime<kmp_uint32>(ident_t *loc, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st) { __kmp_dispatch_init_hierarchy<kmp_uint32>( loc, __kmp_hier_scheds.size, __kmp_hier_scheds.layers, __kmp_hier_scheds.scheds, __kmp_hier_scheds.small_chunks, lb, ub, st); } template <> inline void __kmp_dispatch_init_hier_runtime<kmp_int64>(ident_t *loc, kmp_int64 lb, kmp_int64 ub, kmp_int64 st) { __kmp_dispatch_init_hierarchy<kmp_int64>( loc, __kmp_hier_scheds.size, __kmp_hier_scheds.layers, __kmp_hier_scheds.scheds, __kmp_hier_scheds.large_chunks, lb, ub, st); } template <> inline void __kmp_dispatch_init_hier_runtime<kmp_uint64>(ident_t *loc, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st) { __kmp_dispatch_init_hierarchy<kmp_uint64>( loc, __kmp_hier_scheds.size, __kmp_hier_scheds.layers, __kmp_hier_scheds.scheds, __kmp_hier_scheds.large_chunks, lb, ub, st); } // free all the hierarchy scheduling memory associated with the team void __kmp_dispatch_free_hierarchies(kmp_team_t *team) { int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2; for (int i = 0; i < num_disp_buff; ++i) { // type does not matter here so use kmp_int32 auto sh = reinterpret_cast<dispatch_shared_info_template<kmp_int32> volatile *>( &team->t.t_disp_buffer[i]); if (sh->hier) { sh->hier->deallocate(); __kmp_free(sh->hier); } } } #endif // UT - unsigned flavor of T, ST - signed flavor of T, // DBL - double if sizeof(T)==4, or long double if sizeof(T)==8 template <typename T> static void __kmp_dispatch_init(ident_t *loc, int gtid, enum sched_type schedule, T lb, T ub, typename traits_t<T>::signed_t st, typename traits_t<T>::signed_t chunk, int push_ws) { … } /* For ordered loops, either __kmp_dispatch_finish() should be called after * every iteration, or __kmp_dispatch_finish_chunk() should be called after * every chunk of iterations. If the ordered section(s) were not executed * for this iteration (or every iteration in this chunk), we need to set the * ordered iteration counters so that the next thread can proceed. */ template <typename UT> static void __kmp_dispatch_finish(int gtid, ident_t *loc) { … } #ifdef KMP_GOMP_COMPAT template <typename UT> static void __kmp_dispatch_finish_chunk(int gtid, ident_t *loc) { … } #endif /* KMP_GOMP_COMPAT */ template <typename T> int __kmp_dispatch_next_algorithm(int gtid, dispatch_private_info_template<T> *pr, dispatch_shared_info_template<T> volatile *sh, kmp_int32 *p_last, T *p_lb, T *p_ub, typename traits_t<T>::signed_t *p_st, T nproc, T tid) { … } /* Define a macro for exiting __kmp_dispatch_next(). If status is 0 (no more work), then tell OMPT the loop is over. In some cases kmp_dispatch_fini() is not called. */ #if OMPT_SUPPORT && OMPT_OPTIONAL #define OMPT_LOOP_END … #define OMPT_LOOP_DISPATCH(lb, ub, st, status) … // TODO: implement count #else #define OMPT_LOOP_END … #define OMPT_LOOP_DISPATCH … #endif #if KMP_STATS_ENABLED #define KMP_STATS_LOOP_END … #else #define KMP_STATS_LOOP_END … #endif template <typename T> static int __kmp_dispatch_next(ident_t *loc, int gtid, kmp_int32 *p_last, T *p_lb, T *p_ub, typename traits_t<T>::signed_t *p_st #if OMPT_SUPPORT && OMPT_OPTIONAL , void *codeptr #endif ) { … } /*! @ingroup WORK_SHARING @param loc source location information @param global_tid global thread number @return Zero if the parallel region is not active and this thread should execute all sections, non-zero otherwise. Beginning of sections construct. There are no implicit barriers in the "sections" calls, rather the compiler should introduce an explicit barrier if it is required. This implementation is based on __kmp_dispatch_init, using same constructs for shared data (we can't have sections nested directly in omp for loop, there should be a parallel region in between) */ kmp_int32 __kmpc_sections_init(ident_t *loc, kmp_int32 gtid) { … } /*! @ingroup WORK_SHARING @param loc source location information @param global_tid global thread number @param numberOfSections number of sections in the 'sections' construct @return unsigned [from 0 to n) - number (id) of the section to execute next on this thread. n (or any other number not in range) - nothing to execute on this thread */ kmp_int32 __kmpc_next_section(ident_t *loc, kmp_int32 gtid, kmp_int32 numberOfSections) { … } /*! @ingroup WORK_SHARING @param loc source location information @param global_tid global thread number End of "sections" construct. Don't need to wait here: barrier is added separately when needed. */ void __kmpc_end_sections(ident_t *loc, kmp_int32 gtid) { … } template <typename T> static void __kmp_dist_get_bounds(ident_t *loc, kmp_int32 gtid, kmp_int32 *plastiter, T *plower, T *pupper, typename traits_t<T>::signed_t incr) { … } //----------------------------------------------------------------------------- // Dispatch routines // Transfer call to template< type T > // __kmp_dispatch_init( ident_t *loc, int gtid, enum sched_type schedule, // T lb, T ub, ST st, ST chunk ) extern "C" { /*! @ingroup WORK_SHARING @{ @param loc Source location @param gtid Global thread id @param schedule Schedule type @param lb Lower bound @param ub Upper bound @param st Step (or increment if you prefer) @param chunk The chunk size to block with This function prepares the runtime to start a dynamically scheduled for loop, saving the loop arguments. These functions are all identical apart from the types of the arguments. */ void __kmpc_dispatch_init_4(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk) { … } /*! See @ref __kmpc_dispatch_init_4 */ void __kmpc_dispatch_init_4u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk) { … } /*! See @ref __kmpc_dispatch_init_4 */ void __kmpc_dispatch_init_8(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk) { … } /*! See @ref __kmpc_dispatch_init_4 */ void __kmpc_dispatch_init_8u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk) { … } /*! See @ref __kmpc_dispatch_init_4 Difference from __kmpc_dispatch_init set of functions is these functions are called for composite distribute parallel for construct. Thus before regular iterations dispatching we need to calc per-team iteration space. These functions are all identical apart from the types of the arguments. */ void __kmpc_dist_dispatch_init_4(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk) { … } void __kmpc_dist_dispatch_init_4u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk) { … } void __kmpc_dist_dispatch_init_8(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk) { … } void __kmpc_dist_dispatch_init_8u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 *p_last, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk) { … } /*! @param loc Source code location @param gtid Global thread id @param p_last Pointer to a flag set to one if this is the last chunk or zero otherwise @param p_lb Pointer to the lower bound for the next chunk of work @param p_ub Pointer to the upper bound for the next chunk of work @param p_st Pointer to the stride for the next chunk of work @return one if there is work to be done, zero otherwise Get the next dynamically allocated chunk of work for this thread. If there is no more work, then the lb,ub and stride need not be modified. */ int __kmpc_dispatch_next_4(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_int32 *p_lb, kmp_int32 *p_ub, kmp_int32 *p_st) { … } /*! See @ref __kmpc_dispatch_next_4 */ int __kmpc_dispatch_next_4u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_uint32 *p_lb, kmp_uint32 *p_ub, kmp_int32 *p_st) { … } /*! See @ref __kmpc_dispatch_next_4 */ int __kmpc_dispatch_next_8(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_int64 *p_lb, kmp_int64 *p_ub, kmp_int64 *p_st) { … } /*! See @ref __kmpc_dispatch_next_4 */ int __kmpc_dispatch_next_8u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last, kmp_uint64 *p_lb, kmp_uint64 *p_ub, kmp_int64 *p_st) { … } /*! @param loc Source code location @param gtid Global thread id Mark the end of a dynamic loop. */ void __kmpc_dispatch_fini_4(ident_t *loc, kmp_int32 gtid) { … } /*! See @ref __kmpc_dispatch_fini_4 */ void __kmpc_dispatch_fini_8(ident_t *loc, kmp_int32 gtid) { … } /*! See @ref __kmpc_dispatch_fini_4 */ void __kmpc_dispatch_fini_4u(ident_t *loc, kmp_int32 gtid) { … } /*! See @ref __kmpc_dispatch_fini_4 */ void __kmpc_dispatch_fini_8u(ident_t *loc, kmp_int32 gtid) { … } /*! See @ref __kmpc_dispatch_deinit */ void __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 gtid) { … } /*! @} */ //----------------------------------------------------------------------------- // Non-template routines from kmp_dispatch.cpp used in other sources kmp_uint32 __kmp_eq_4(kmp_uint32 value, kmp_uint32 checker) { … } kmp_uint32 __kmp_neq_4(kmp_uint32 value, kmp_uint32 checker) { … } kmp_uint32 __kmp_lt_4(kmp_uint32 value, kmp_uint32 checker) { … } kmp_uint32 __kmp_ge_4(kmp_uint32 value, kmp_uint32 checker) { … } kmp_uint32 __kmp_le_4(kmp_uint32 value, kmp_uint32 checker) { … } kmp_uint32 __kmp_wait_4(volatile kmp_uint32 *spinner, kmp_uint32 checker, kmp_uint32 (*pred)(kmp_uint32, kmp_uint32), void *obj // Higher-level synchronization object, or NULL. ) { … } void __kmp_wait_4_ptr(void *spinner, kmp_uint32 checker, kmp_uint32 (*pred)(void *, kmp_uint32), void *obj // Higher-level synchronization object, or NULL. ) { … } } // extern "C" #ifdef KMP_GOMP_COMPAT void __kmp_aux_dispatch_init_4(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int32 lb, kmp_int32 ub, kmp_int32 st, kmp_int32 chunk, int push_ws) { … } void __kmp_aux_dispatch_init_4u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint32 lb, kmp_uint32 ub, kmp_int32 st, kmp_int32 chunk, int push_ws) { … } void __kmp_aux_dispatch_init_8(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_int64 lb, kmp_int64 ub, kmp_int64 st, kmp_int64 chunk, int push_ws) { … } void __kmp_aux_dispatch_init_8u(ident_t *loc, kmp_int32 gtid, enum sched_type schedule, kmp_uint64 lb, kmp_uint64 ub, kmp_int64 st, kmp_int64 chunk, int push_ws) { … } void __kmp_aux_dispatch_fini_chunk_4(ident_t *loc, kmp_int32 gtid) { … } void __kmp_aux_dispatch_fini_chunk_8(ident_t *loc, kmp_int32 gtid) { … } void __kmp_aux_dispatch_fini_chunk_4u(ident_t *loc, kmp_int32 gtid) { … } void __kmp_aux_dispatch_fini_chunk_8u(ident_t *loc, kmp_int32 gtid) { … } #endif /* KMP_GOMP_COMPAT */ /* ------------------------------------------------------------------------ */
Codebase Browser
llvm