/* * kmp_dispatch.h: dynamic scheduling - iteration initialization and dispatch. */ //===----------------------------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef KMP_DISPATCH_H #define KMP_DISPATCH_H /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ #include "kmp.h" #include "kmp_error.h" #include "kmp_i18n.h" #include "kmp_itt.h" #include "kmp_stats.h" #include "kmp_str.h" #if KMP_OS_WINDOWS && KMP_ARCH_X86 #include <float.h> #endif #if OMPT_SUPPORT #include "ompt-internal.h" #include "ompt-specific.h" #endif /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ #if KMP_USE_HIER_SCHED // Forward declarations of some hierarchical scheduling data structures template <typename T> struct kmp_hier_t; template <typename T> struct kmp_hier_top_unit_t; #endif // KMP_USE_HIER_SCHED template <typename T> struct dispatch_shared_info_template; template <typename T> struct dispatch_private_info_template; template <typename T> extern void __kmp_dispatch_init_algorithm(ident_t *loc, int gtid, dispatch_private_info_template<T> *pr, enum sched_type schedule, T lb, T ub, typename traits_t<T>::signed_t st, #if USE_ITT_BUILD kmp_uint64 *cur_chunk, #endif typename traits_t<T>::signed_t chunk, T nproc, T unit_id); template <typename T> extern int __kmp_dispatch_next_algorithm( int gtid, dispatch_private_info_template<T> *pr, dispatch_shared_info_template<T> volatile *sh, kmp_int32 *p_last, T *p_lb, T *p_ub, typename traits_t<T>::signed_t *p_st, T nproc, T unit_id); void __kmp_dispatch_dxo_error(int *gtid_ref, int *cid_ref, ident_t *loc_ref); void __kmp_dispatch_deo_error(int *gtid_ref, int *cid_ref, ident_t *loc_ref); #if KMP_STATIC_STEAL_ENABLED // replaces dispatch_private_info{32,64} structures and // dispatch_private_info{32,64}_t types template <typename T> struct dispatch_private_infoXX_template { … }; #else /* KMP_STATIC_STEAL_ENABLED */ // replaces dispatch_private_info{32,64} structures and // dispatch_private_info{32,64}_t types template <typename T> struct dispatch_private_infoXX_template { typedef typename traits_t<T>::unsigned_t UT; typedef typename traits_t<T>::signed_t ST; T lb; T ub; ST st; // signed UT tc; // unsigned T parm1; T parm2; T parm3; T parm4; UT count; // unsigned UT ordered_lower; // unsigned UT ordered_upper; // unsigned #if KMP_OS_WINDOWS T last_upper; #endif /* KMP_OS_WINDOWS */ }; #endif /* KMP_STATIC_STEAL_ENABLED */ template <typename T> struct KMP_ALIGN_CACHE dispatch_private_info_template { … }; // replaces dispatch_shared_info{32,64} structures and // dispatch_shared_info{32,64}_t types template <typename T> struct dispatch_shared_infoXX_template { … }; // replaces dispatch_shared_info structure and dispatch_shared_info_t type template <typename T> struct dispatch_shared_info_template { … }; /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ #undef USE_TEST_LOCKS // test_then_add template (general template should NOT be used) template <typename T> static __forceinline T test_then_add(volatile T *p, T d); template <> __forceinline kmp_int32 test_then_add<kmp_int32>(volatile kmp_int32 *p, kmp_int32 d) { … } template <> __forceinline kmp_int64 test_then_add<kmp_int64>(volatile kmp_int64 *p, kmp_int64 d) { … } // test_then_inc_acq template (general template should NOT be used) template <typename T> static __forceinline T test_then_inc_acq(volatile T *p); template <> __forceinline kmp_int32 test_then_inc_acq<kmp_int32>(volatile kmp_int32 *p) { … } template <> __forceinline kmp_int64 test_then_inc_acq<kmp_int64>(volatile kmp_int64 *p) { … } // test_then_inc template (general template should NOT be used) template <typename T> static __forceinline T test_then_inc(volatile T *p); template <> __forceinline kmp_int32 test_then_inc<kmp_int32>(volatile kmp_int32 *p) { … } template <> __forceinline kmp_int64 test_then_inc<kmp_int64>(volatile kmp_int64 *p) { … } // compare_and_swap template (general template should NOT be used) template <typename T> static __forceinline kmp_int32 compare_and_swap(volatile T *p, T c, T s); template <> __forceinline kmp_int32 compare_and_swap<kmp_int32>(volatile kmp_int32 *p, kmp_int32 c, kmp_int32 s) { … } template <> __forceinline kmp_int32 compare_and_swap<kmp_int64>(volatile kmp_int64 *p, kmp_int64 c, kmp_int64 s) { … } template <typename T> kmp_uint32 __kmp_ge(T value, T checker) { … } template <typename T> kmp_uint32 __kmp_eq(T value, T checker) { … } /* Spin wait loop that pauses between checks. Waits until function returns non-zero when called with *spinner and check. Does NOT put threads to sleep. Arguments: UT is unsigned 4- or 8-byte type spinner - memory location to check value checker - value which spinner is >, <, ==, etc. pred - predicate function to perform binary comparison of some sort #if USE_ITT_BUILD obj -- is higher-level synchronization object to report to ittnotify. It is used to report locks consistently. For example, if lock is acquired immediately, its address is reported to ittnotify via KMP_FSYNC_ACQUIRED(). However, it lock cannot be acquired immediately and lock routine calls to KMP_WAIT(), the later should report the same address, not an address of low-level spinner. #endif // USE_ITT_BUILD TODO: make inline function (move to header file for icl) */ template <typename UT> static UT __kmp_wait(volatile UT *spinner, UT checker, kmp_uint32 (*pred)(UT, UT) USE_ITT_BUILD_ARG(void *obj)) { … } /* ------------------------------------------------------------------------ */ /* ------------------------------------------------------------------------ */ template <typename UT> void __kmp_dispatch_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { … } template <typename UT> void __kmp_dispatch_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) { … } /* Computes and returns x to the power of y, where y must a non-negative integer */ template <typename UT> static __forceinline long double __kmp_pow(long double x, UT y) { … } /* Computes and returns the number of unassigned iterations after idx chunks have been assigned (the total number of unassigned iterations in chunks with index greater than or equal to idx). __forceinline seems to be broken so that if we __forceinline this function, the behavior is wrong (one of the unit tests, sch_guided_analytical_basic.cpp, fails) */ template <typename T> static __inline typename traits_t<T>::unsigned_t __kmp_dispatch_guided_remaining(T tc, typename traits_t<T>::floating_t base, typename traits_t<T>::unsigned_t idx) { … } // Parameters of the guided-iterative algorithm: // p2 = n * nproc * ( chunk + 1 ) // point of switching to dynamic // p3 = 1 / ( n * nproc ) // remaining iterations multiplier // by default n = 2. For example with n = 3 the chunks distribution will be more // flat. // With n = 1 first chunk is the same as for static schedule, e.g. trip / nproc. static const int guided_int_param = …; static const double guided_flt_param = …; // = 1.0 / guided_int_param; #endif // KMP_DISPATCH_H
Codebase Browser
llvm