/*
* z_Windows_NT_util.cpp -- platform specific routines.
*/
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "kmp.h"
#include "kmp_affinity.h"
#include "kmp_i18n.h"
#include "kmp_io.h"
#include "kmp_itt.h"
#include "kmp_wait_release.h"
/* This code is related to NtQuerySystemInformation() function. This function
is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
number of running threads in the system. */
#include <ntsecapi.h> // UNICODE_STRING
#undef WIN32_NO_STATUS
#include <ntstatus.h>
#include <psapi.h>
#ifdef _MSC_VER
#pragma comment(lib, "psapi.lib")
#endif
enum SYSTEM_INFORMATION_CLASS {
SystemProcessInformation = 5
}; // SYSTEM_INFORMATION_CLASS
struct CLIENT_ID {
HANDLE UniqueProcess;
HANDLE UniqueThread;
}; // struct CLIENT_ID
enum THREAD_STATE {
StateInitialized,
StateReady,
StateRunning,
StateStandby,
StateTerminated,
StateWait,
StateTransition,
StateUnknown
}; // enum THREAD_STATE
struct VM_COUNTERS {
SIZE_T PeakVirtualSize;
SIZE_T VirtualSize;
ULONG PageFaultCount;
SIZE_T PeakWorkingSetSize;
SIZE_T WorkingSetSize;
SIZE_T QuotaPeakPagedPoolUsage;
SIZE_T QuotaPagedPoolUsage;
SIZE_T QuotaPeakNonPagedPoolUsage;
SIZE_T QuotaNonPagedPoolUsage;
SIZE_T PagefileUsage;
SIZE_T PeakPagefileUsage;
SIZE_T PrivatePageCount;
}; // struct VM_COUNTERS
struct SYSTEM_THREAD {
LARGE_INTEGER KernelTime;
LARGE_INTEGER UserTime;
LARGE_INTEGER CreateTime;
ULONG WaitTime;
LPVOID StartAddress;
CLIENT_ID ClientId;
DWORD Priority;
LONG BasePriority;
ULONG ContextSwitchCount;
THREAD_STATE State;
ULONG WaitReason;
}; // SYSTEM_THREAD
KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
#if KMP_ARCH_X86 || KMP_ARCH_ARM
KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
#else
KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
#endif
struct SYSTEM_PROCESS_INFORMATION {
ULONG NextEntryOffset;
ULONG NumberOfThreads;
LARGE_INTEGER Reserved[3];
LARGE_INTEGER CreateTime;
LARGE_INTEGER UserTime;
LARGE_INTEGER KernelTime;
UNICODE_STRING ImageName;
DWORD BasePriority;
HANDLE ProcessId;
HANDLE ParentProcessId;
ULONG HandleCount;
ULONG Reserved2[2];
VM_COUNTERS VMCounters;
IO_COUNTERS IOCounters;
SYSTEM_THREAD Threads[1];
}; // SYSTEM_PROCESS_INFORMATION
typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
#if KMP_ARCH_X86 || KMP_ARCH_ARM
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
#else
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
#endif
typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
PVOID, ULONG, PULONG);
NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
HMODULE ntdll = NULL;
/* End of NtQuerySystemInformation()-related code */
static HMODULE kernel32 = NULL;
#if KMP_HANDLE_SIGNALS
typedef void (*sig_func_t)(int);
static sig_func_t __kmp_sighldrs[NSIG];
static int __kmp_siginstalled[NSIG];
#endif
#if KMP_USE_MONITOR
static HANDLE __kmp_monitor_ev;
#endif
static kmp_int64 __kmp_win32_time;
double __kmp_win32_tick;
int __kmp_init_runtime = FALSE;
CRITICAL_SECTION __kmp_win32_section;
void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
InitializeCriticalSection(&mx->cs);
#if USE_ITT_BUILD
__kmp_itt_system_object_created(&mx->cs, "Critical Section");
#endif /* USE_ITT_BUILD */
}
void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
DeleteCriticalSection(&mx->cs);
}
void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
EnterCriticalSection(&mx->cs);
}
int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
return TryEnterCriticalSection(&mx->cs);
}
void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
LeaveCriticalSection(&mx->cs);
}
void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
cv->waiters_count_ = 0;
cv->wait_generation_count_ = 0;
cv->release_count_ = 0;
/* Initialize the critical section */
__kmp_win32_mutex_init(&cv->waiters_count_lock_);
/* Create a manual-reset event. */
cv->event_ = CreateEvent(NULL, // no security
TRUE, // manual-reset
FALSE, // non-signaled initially
NULL); // unnamed
#if USE_ITT_BUILD
__kmp_itt_system_object_created(cv->event_, "Event");
#endif /* USE_ITT_BUILD */
}
void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
__kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
__kmp_free_handle(cv->event_);
memset(cv, '\0', sizeof(*cv));
}
/* TODO associate cv with a team instead of a thread so as to optimize
the case where we wake up a whole team */
template <class C>
static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
kmp_info_t *th, C *flag) {
int my_generation;
int last_waiter;
/* Avoid race conditions */
__kmp_win32_mutex_lock(&cv->waiters_count_lock_);
/* Increment count of waiters */
cv->waiters_count_++;
/* Store current generation in our activation record. */
my_generation = cv->wait_generation_count_;
__kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
__kmp_win32_mutex_unlock(mx);
for (;;) {
int wait_done = 0;
DWORD res, timeout = 5000; // just tried to quess an appropriate number
/* Wait until the event is signaled */
res = WaitForSingleObject(cv->event_, timeout);
if (res == WAIT_OBJECT_0) {
// event signaled
__kmp_win32_mutex_lock(&cv->waiters_count_lock_);
/* Exit the loop when the <cv->event_> is signaled and there are still
waiting threads from this <wait_generation> that haven't been released
from this wait yet. */
wait_done = (cv->release_count_ > 0) &&
(cv->wait_generation_count_ != my_generation);
__kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
} else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) {
// check if the flag and cv counters are in consistent state
// as MS sent us debug dump whith inconsistent state of data
__kmp_win32_mutex_lock(mx);
typename C::flag_t old_f = flag->set_sleeping();
if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) {
__kmp_win32_mutex_unlock(mx);
continue;
}
// condition fulfilled, exiting
flag->unset_sleeping();
TCW_PTR(th->th.th_sleep_loc, NULL);
th->th.th_sleep_loc_type = flag_unset;
KF_TRACE(50, ("__kmp_win32_cond_wait: exiting, condition "
"fulfilled: flag's loc(%p): %u\n",
flag->get(), (unsigned int)flag->load()));
__kmp_win32_mutex_lock(&cv->waiters_count_lock_);
KMP_DEBUG_ASSERT(cv->waiters_count_ > 0);
cv->release_count_ = cv->waiters_count_;
cv->wait_generation_count_++;
wait_done = 1;
__kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
__kmp_win32_mutex_unlock(mx);
}
/* there used to be a semicolon after the if statement, it looked like a
bug, so i removed it */
if (wait_done)
break;
}
__kmp_win32_mutex_lock(mx);
__kmp_win32_mutex_lock(&cv->waiters_count_lock_);
cv->waiters_count_--;
cv->release_count_--;
last_waiter = (cv->release_count_ == 0);
__kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
if (last_waiter) {
/* We're the last waiter to be notified, so reset the manual event. */
ResetEvent(cv->event_);
}
}
void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
__kmp_win32_mutex_lock(&cv->waiters_count_lock_);
if (cv->waiters_count_ > 0) {
SetEvent(cv->event_);
/* Release all the threads in this generation. */
cv->release_count_ = cv->waiters_count_;
/* Start a new generation. */
cv->wait_generation_count_++;
}
__kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
}
void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
__kmp_win32_cond_broadcast(cv);
}
void __kmp_enable(int new_state) {
if (__kmp_init_runtime)
LeaveCriticalSection(&__kmp_win32_section);
}
void __kmp_disable(int *old_state) {
*old_state = 0;
if (__kmp_init_runtime)
EnterCriticalSection(&__kmp_win32_section);
}
void __kmp_suspend_initialize(void) { /* do nothing */
}
void __kmp_suspend_initialize_thread(kmp_info_t *th) {
int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init);
int new_value = TRUE;
// Return if already initialized
if (old_value == new_value)
return;
// Wait, then return if being initialized
if (old_value == -1 ||
!__kmp_atomic_compare_store(&th->th.th_suspend_init, old_value, -1)) {
while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init) != new_value) {
KMP_CPU_PAUSE();
}
} else {
// Claim to be the initializer and do initializations
__kmp_win32_cond_init(&th->th.th_suspend_cv);
__kmp_win32_mutex_init(&th->th.th_suspend_mx);
KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, new_value);
}
}
void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init)) {
/* this means we have initialize the suspension pthread objects for this
thread in this instance of the process */
__kmp_win32_cond_destroy(&th->th.th_suspend_cv);
__kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, FALSE);
}
}
int __kmp_try_suspend_mx(kmp_info_t *th) {
return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
}
void __kmp_lock_suspend_mx(kmp_info_t *th) {
__kmp_win32_mutex_lock(&th->th.th_suspend_mx);
}
void __kmp_unlock_suspend_mx(kmp_info_t *th) {
__kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
}
/* This routine puts the calling thread to sleep after setting the
sleep bit for the indicated flag variable to true. */
template <class C>
static inline void __kmp_suspend_template(int th_gtid, C *flag) {
kmp_info_t *th = __kmp_threads[th_gtid];
typename C::flag_t old_spin;
KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
th_gtid, flag->get()));
__kmp_suspend_initialize_thread(th);
__kmp_lock_suspend_mx(th);
KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
" loc(%p)\n",
th_gtid, flag->get()));
/* TODO: shouldn't this use release semantics to ensure that
__kmp_suspend_initialize_thread gets called first? */
old_spin = flag->set_sleeping();
TCW_PTR(th->th.th_sleep_loc, (void *)flag);
th->th.th_sleep_loc_type = flag->get_type();
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
__kmp_pause_status != kmp_soft_paused) {
flag->unset_sleeping();
TCW_PTR(th->th.th_sleep_loc, NULL);
th->th.th_sleep_loc_type = flag_unset;
__kmp_unlock_suspend_mx(th);
return;
}
KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
" loc(%p)==%u\n",
th_gtid, flag->get(), (unsigned int)flag->load()));
if (flag->done_check_val(old_spin) || flag->done_check()) {
flag->unset_sleeping();
TCW_PTR(th->th.th_sleep_loc, NULL);
th->th.th_sleep_loc_type = flag_unset;
KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
"for flag's loc(%p)\n",
th_gtid, flag->get()));
} else {
#ifdef DEBUG_SUSPEND
__kmp_suspend_count++;
#endif
/* Encapsulate in a loop as the documentation states that this may "with
low probability" return when the condition variable has not been signaled
or broadcast */
int deactivated = FALSE;
while (flag->is_sleeping()) {
KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
"kmp_win32_cond_wait()\n",
th_gtid));
// Mark the thread as no longer active (only in the first iteration of the
// loop).
if (!deactivated) {
th->th.th_active = FALSE;
if (th->th.th_active_in_pool) {
th->th.th_active_in_pool = FALSE;
KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
}
deactivated = TRUE;
}
KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
KMP_DEBUG_ASSERT(th->th.th_sleep_loc_type == flag->get_type());
__kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
flag);
#ifdef KMP_DEBUG
if (flag->is_sleeping()) {
KF_TRACE(100,
("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
}
#endif /* KMP_DEBUG */
} // while
// We may have had the loop variable set before entering the loop body;
// so we need to reset sleep_loc.
TCW_PTR(th->th.th_sleep_loc, NULL);
th->th.th_sleep_loc_type = flag_unset;
KMP_DEBUG_ASSERT(!flag->is_sleeping());
KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
// Mark the thread as active again (if it was previous marked as inactive)
if (deactivated) {
th->th.th_active = TRUE;
if (TCR_4(th->th.th_in_pool)) {
KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
th->th.th_active_in_pool = TRUE;
}
}
}
__kmp_unlock_suspend_mx(th);
KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
}
template <bool C, bool S>
void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
__kmp_suspend_template(th_gtid, flag);
}
template <bool C, bool S>
void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
__kmp_suspend_template(th_gtid, flag);
}
template <bool C, bool S>
void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
__kmp_suspend_template(th_gtid, flag);
}
void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
__kmp_suspend_template(th_gtid, flag);
}
template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
template void
__kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
template void
__kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
/* This routine signals the thread specified by target_gtid to wake up
after setting the sleep bit indicated by the flag argument to FALSE */
template <class C>
static inline void __kmp_resume_template(int target_gtid, C *flag) {
kmp_info_t *th = __kmp_threads[target_gtid];
#ifdef KMP_DEBUG
int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
#endif
KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
gtid, target_gtid));
__kmp_suspend_initialize_thread(th);
__kmp_lock_suspend_mx(th);
if (!flag || flag != th->th.th_sleep_loc) {
// coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
// different location; wake up at new location
flag = (C *)th->th.th_sleep_loc;
}
// First, check if the flag is null or its type has changed. If so, someone
// else woke it up.
if (!flag || flag->get_type() != th->th.th_sleep_loc_type) {
// simply shows what flag was cast to
KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
"awake: flag's loc(%p)\n",
gtid, target_gtid, NULL));
__kmp_unlock_suspend_mx(th);
return;
} else {
if (!flag->is_sleeping()) {
KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
"awake: flag's loc(%p): %u\n",
gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
__kmp_unlock_suspend_mx(th);
return;
}
}
KMP_DEBUG_ASSERT(flag);
flag->unset_sleeping();
TCW_PTR(th->th.th_sleep_loc, NULL);
th->th.th_sleep_loc_type = flag_unset;
KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
"bit for flag's loc(%p)\n",
gtid, target_gtid, flag->get()));
__kmp_win32_cond_signal(&th->th.th_suspend_cv);
__kmp_unlock_suspend_mx(th);
KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
" for T#%d\n",
gtid, target_gtid));
}
template <bool C, bool S>
void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
__kmp_resume_template(target_gtid, flag);
}
template <bool C, bool S>
void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
__kmp_resume_template(target_gtid, flag);
}
template <bool C, bool S>
void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
__kmp_resume_template(target_gtid, flag);
}
void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
__kmp_resume_template(target_gtid, flag);
}
template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
template void
__kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
void __kmp_yield() { Sleep(0); }
void __kmp_gtid_set_specific(int gtid) {
if (__kmp_init_gtid) {
KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
__kmp_gtid_threadprivate_key));
kmp_intptr_t g = (kmp_intptr_t)gtid;
if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(g + 1)))
KMP_FATAL(TLSSetValueFailed);
} else {
KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
}
}
int __kmp_gtid_get_specific() {
int gtid;
if (!__kmp_init_gtid) {
KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
"KMP_GTID_SHUTDOWN\n"));
return KMP_GTID_SHUTDOWN;
}
gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
if (gtid == 0) {
gtid = KMP_GTID_DNE;
} else {
gtid--;
}
KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
__kmp_gtid_threadprivate_key, gtid));
return gtid;
}
void __kmp_affinity_bind_thread(int proc) {
if (__kmp_num_proc_groups > 1) {
// Form the GROUP_AFFINITY struct directly, rather than filling
// out a bit vector and calling __kmp_set_system_affinity().
GROUP_AFFINITY ga;
KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
sizeof(DWORD_PTR))));
ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
DWORD error = GetLastError();
// AC: continue silently if not verbose
if (__kmp_affinity.flags.verbose) {
kmp_msg_t err_code = KMP_ERR(error);
__kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
__kmp_msg_null);
if (__kmp_generate_warnings == kmp_warnings_off) {
__kmp_str_free(&err_code.str);
}
}
}
} else {
kmp_affin_mask_t *mask;
KMP_CPU_ALLOC_ON_STACK(mask);
KMP_CPU_ZERO(mask);
KMP_CPU_SET(proc, mask);
__kmp_set_system_affinity(mask, TRUE);
KMP_CPU_FREE_FROM_STACK(mask);
}
}
void __kmp_affinity_determine_capable(const char *env_var) {
// All versions of Windows* OS (since Win '95) support
// SetThreadAffinityMask().
#if KMP_GROUP_AFFINITY
KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
#else
KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
#endif
KA_TRACE(10, ("__kmp_affinity_determine_capable: "
"Windows* OS affinity interface functional (mask size = "
"%" KMP_SIZE_T_SPEC ").\n",
__kmp_affin_mask_size));
}
double __kmp_read_cpu_time(void) {
FILETIME CreationTime, ExitTime, KernelTime, UserTime;
int status;
double cpu_time;
cpu_time = 0;
status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
&KernelTime, &UserTime);
if (status) {
double sec = 0;
sec += KernelTime.dwHighDateTime;
sec += UserTime.dwHighDateTime;
/* Shift left by 32 bits */
sec *= (double)(1 << 16) * (double)(1 << 16);
sec += KernelTime.dwLowDateTime;
sec += UserTime.dwLowDateTime;
cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
}
return cpu_time;
}
int __kmp_read_system_info(struct kmp_sys_info *info) {
info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
info->minflt = 0; /* the number of page faults serviced without any I/O */
info->majflt = 0; /* the number of page faults serviced that required I/O */
info->nswap = 0; // the number of times a process was "swapped" out of memory
info->inblock = 0; // the number of times the file system had to perform input
info->oublock = 0; // number of times the file system had to perform output
info->nvcsw = 0; /* the number of times a context switch was voluntarily */
info->nivcsw = 0; /* the number of times a context switch was forced */
return 1;
}
void __kmp_runtime_initialize(void) {
SYSTEM_INFO info;
kmp_str_buf_t path;
UINT path_size;
if (__kmp_init_runtime) {
return;
}
#if KMP_DYNAMIC_LIB
/* Pin dynamic library for the lifetime of application */
{
// First, turn off error message boxes
UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
HMODULE h;
BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
GET_MODULE_HANDLE_EX_FLAG_PIN,
(LPCTSTR)&__kmp_serial_initialize, &h);
(void)ret;
KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
SetErrorMode(err_mode); // Restore error mode
KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
}
#endif
InitializeCriticalSection(&__kmp_win32_section);
#if USE_ITT_BUILD
__kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
#endif /* USE_ITT_BUILD */
__kmp_initialize_system_tick();
#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
if (!__kmp_cpuinfo.initialized) {
__kmp_query_cpuid(&__kmp_cpuinfo);
}
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
/* Set up minimum number of threads to switch to TLS gtid */
#if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
// Windows* OS, static library.
/* New thread may use stack space previously used by another thread,
currently terminated. On Windows* OS, in case of static linking, we do not
know the moment of thread termination, and our structures (__kmp_threads
and __kmp_root arrays) are still keep info about dead threads. This leads
to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
(by searching through stack addresses of all known threads) for
unregistered foreign tread.
Setting __kmp_tls_gtid_min to 0 workarounds this problem:
__kmp_get_global_thread_id() does not search through stacks, but get gtid
from TLS immediately.
--ln
*/
__kmp_tls_gtid_min = 0;
#else
__kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
#endif
/* for the static library */
if (!__kmp_gtid_threadprivate_key) {
__kmp_gtid_threadprivate_key = TlsAlloc();
if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
KMP_FATAL(TLSOutOfIndexes);
}
}
// Load ntdll.dll.
/* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
(see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
have to specify full path to the library. */
__kmp_str_buf_init(&path);
path_size = GetSystemDirectory(path.str, path.size);
KMP_DEBUG_ASSERT(path_size > 0);
if (path_size >= path.size) {
// Buffer is too short. Expand the buffer and try again.
__kmp_str_buf_reserve(&path, path_size);
path_size = GetSystemDirectory(path.str, path.size);
KMP_DEBUG_ASSERT(path_size > 0);
}
if (path_size > 0 && path_size < path.size) {
// Now we have system directory name in the buffer.
// Append backslash and name of dll to form full path,
path.used = path_size;
__kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
// Now load ntdll using full path.
ntdll = GetModuleHandle(path.str);
}
KMP_DEBUG_ASSERT(ntdll != NULL);
if (ntdll != NULL) {
NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
ntdll, "NtQuerySystemInformation");
}
KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
#if KMP_GROUP_AFFINITY
// Load kernel32.dll.
// Same caveat - must use full system path name.
if (path_size > 0 && path_size < path.size) {
// Truncate the buffer back to just the system path length,
// discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
path.used = path_size;
__kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
// Load kernel32.dll using full path.
kernel32 = GetModuleHandle(path.str);
KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
// Load the function pointers to kernel32.dll routines
// that may or may not exist on this system.
if (kernel32 != NULL) {
__kmp_GetActiveProcessorCount =
(kmp_GetActiveProcessorCount_t)GetProcAddress(
kernel32, "GetActiveProcessorCount");
__kmp_GetActiveProcessorGroupCount =
(kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
kernel32, "GetActiveProcessorGroupCount");
__kmp_GetThreadGroupAffinity =
(kmp_GetThreadGroupAffinity_t)GetProcAddress(
kernel32, "GetThreadGroupAffinity");
__kmp_SetThreadGroupAffinity =
(kmp_SetThreadGroupAffinity_t)GetProcAddress(
kernel32, "SetThreadGroupAffinity");
KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
" = %p\n",
__kmp_GetActiveProcessorCount));
KA_TRACE(10, ("__kmp_runtime_initialize: "
"__kmp_GetActiveProcessorGroupCount = %p\n",
__kmp_GetActiveProcessorGroupCount));
KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
" = %p\n",
__kmp_GetThreadGroupAffinity));
KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
" = %p\n",
__kmp_SetThreadGroupAffinity));
KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
sizeof(kmp_affin_mask_t)));
// See if group affinity is supported on this system.
// If so, calculate the #groups and #procs.
//
// Group affinity was introduced with Windows* 7 OS and
// Windows* Server 2008 R2 OS.
if ((__kmp_GetActiveProcessorCount != NULL) &&
(__kmp_GetActiveProcessorGroupCount != NULL) &&
(__kmp_GetThreadGroupAffinity != NULL) &&
(__kmp_SetThreadGroupAffinity != NULL) &&
((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
1)) {
// Calculate the total number of active OS procs.
int i;
KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
" detected\n",
__kmp_num_proc_groups));
__kmp_xproc = 0;
for (i = 0; i < __kmp_num_proc_groups; i++) {
DWORD size = __kmp_GetActiveProcessorCount(i);
__kmp_xproc += size;
KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
i, size));
}
} else {
KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
" detected\n",
__kmp_num_proc_groups));
}
}
}
if (__kmp_num_proc_groups <= 1) {
GetSystemInfo(&info);
__kmp_xproc = info.dwNumberOfProcessors;
}
#else
(void)kernel32;
GetSystemInfo(&info);
__kmp_xproc = info.dwNumberOfProcessors;
#endif /* KMP_GROUP_AFFINITY */
// If the OS said there were 0 procs, take a guess and use a value of 2.
// This is done for Linux* OS, also. Do we need error / warning?
if (__kmp_xproc <= 0) {
__kmp_xproc = 2;
}
KA_TRACE(5,
("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
__kmp_str_buf_free(&path);
#if USE_ITT_BUILD
__kmp_itt_initialize();
#endif /* USE_ITT_BUILD */
__kmp_init_runtime = TRUE;
} // __kmp_runtime_initialize
void __kmp_runtime_destroy(void) {
if (!__kmp_init_runtime) {
return;
}
#if USE_ITT_BUILD
__kmp_itt_destroy();
#endif /* USE_ITT_BUILD */
/* we can't DeleteCriticalsection( & __kmp_win32_section ); */
/* due to the KX_TRACE() commands */
KA_TRACE(40, ("__kmp_runtime_destroy\n"));
if (__kmp_gtid_threadprivate_key) {
TlsFree(__kmp_gtid_threadprivate_key);
__kmp_gtid_threadprivate_key = 0;
}
__kmp_affinity_uninitialize();
DeleteCriticalSection(&__kmp_win32_section);
ntdll = NULL;
NtQuerySystemInformation = NULL;
#if KMP_ARCH_X86_64
kernel32 = NULL;
__kmp_GetActiveProcessorCount = NULL;
__kmp_GetActiveProcessorGroupCount = NULL;
__kmp_GetThreadGroupAffinity = NULL;
__kmp_SetThreadGroupAffinity = NULL;
#endif // KMP_ARCH_X86_64
__kmp_init_runtime = FALSE;
}
void __kmp_terminate_thread(int gtid) {
kmp_info_t *th = __kmp_threads[gtid];
if (!th)
return;
KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
/* It's OK, the thread may have exited already */
}
__kmp_free_handle(th->th.th_info.ds.ds_thread);
}
void __kmp_clear_system_time(void) {
LARGE_INTEGER time;
QueryPerformanceCounter(&time);
__kmp_win32_time = (kmp_int64)time.QuadPart;
}
void __kmp_initialize_system_tick(void) {
{
BOOL status;
LARGE_INTEGER freq;
status = QueryPerformanceFrequency(&freq);
if (!status) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
KMP_ERR(error), __kmp_msg_null);
} else {
__kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
}
}
}
/* Calculate the elapsed wall clock time for the user */
void __kmp_elapsed(double *t) {
LARGE_INTEGER now;
QueryPerformanceCounter(&now);
*t = ((double)now.QuadPart) * __kmp_win32_tick;
}
/* Calculate the elapsed wall clock tick for the user */
void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
void __kmp_read_system_time(double *delta) {
if (delta != NULL) {
LARGE_INTEGER now;
QueryPerformanceCounter(&now);
*delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
__kmp_win32_tick;
}
}
/* Return the current time stamp in nsec */
kmp_uint64 __kmp_now_nsec() {
LARGE_INTEGER now;
QueryPerformanceCounter(&now);
return 1e9 * __kmp_win32_tick * now.QuadPart;
}
extern "C" void *__stdcall __kmp_launch_worker(void *arg) {
volatile void *stack_data;
void *exit_val;
void *padding = 0;
kmp_info_t *this_thr = (kmp_info_t *)arg;
int gtid;
gtid = this_thr->th.th_info.ds.ds_gtid;
__kmp_gtid_set_specific(gtid);
#ifdef KMP_TDATA_GTID
#error "This define causes problems with LoadLibrary() + declspec(thread) " \
"on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
"reference: http://support.microsoft.com/kb/118816"
//__kmp_gtid = gtid;
#endif
#if USE_ITT_BUILD
__kmp_itt_thread_name(gtid);
#endif /* USE_ITT_BUILD */
__kmp_affinity_bind_init_mask(gtid);
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
// Set FP control regs to be a copy of the parallel initialization thread's.
__kmp_clear_x87_fpu_status_word();
__kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
__kmp_load_mxcsr(&__kmp_init_mxcsr);
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
if (__kmp_stkoffset > 0 && gtid > 0) {
padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
(void)padding;
}
KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
if (TCR_4(__kmp_gtid_mode) <
2) { // check stack only if it is used to get gtid
TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
__kmp_check_stack_overlap(this_thr);
}
KMP_MB();
exit_val = __kmp_launch_thread(this_thr);
KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
KMP_MB();
return exit_val;
}
#if KMP_USE_MONITOR
/* The monitor thread controls all of the threads in the complex */
void *__stdcall __kmp_launch_monitor(void *arg) {
DWORD wait_status;
kmp_thread_t monitor;
int status;
int interval;
kmp_info_t *this_thr = (kmp_info_t *)arg;
KMP_DEBUG_ASSERT(__kmp_init_monitor);
TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
// TODO: hide "2" in enum (like {true,false,started})
this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
KMP_MB(); /* Flush all pending memory write invalidates. */
KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
monitor = GetCurrentThread();
/* set thread priority */
status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
if (!status) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
}
/* register us as monitor */
__kmp_gtid_set_specific(KMP_GTID_MONITOR);
#ifdef KMP_TDATA_GTID
#error "This define causes problems with LoadLibrary() + declspec(thread) " \
"on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
"reference: http://support.microsoft.com/kb/118816"
//__kmp_gtid = KMP_GTID_MONITOR;
#endif
#if USE_ITT_BUILD
__kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
// monitor thread.
#endif /* USE_ITT_BUILD */
KMP_MB(); /* Flush all pending memory write invalidates. */
interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
while (!TCR_4(__kmp_global.g.g_done)) {
/* This thread monitors the state of the system */
KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
if (wait_status == WAIT_TIMEOUT) {
TCW_4(__kmp_global.g.g_time.dt.t_value,
TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
}
KMP_MB(); /* Flush all pending memory write invalidates. */
}
KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
if (!status) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
}
if (__kmp_global.g.g_abort != 0) {
/* now we need to terminate the worker threads */
/* the value of t_abort is the signal we caught */
int gtid;
KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
(__kmp_global.g.g_abort)));
/* terminate the OpenMP worker threads */
/* TODO this is not valid for sibling threads!!
* the uber master might not be 0 anymore.. */
for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
__kmp_terminate_thread(gtid);
__kmp_cleanup();
Sleep(0);
KA_TRACE(10,
("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
if (__kmp_global.g.g_abort > 0) {
raise(__kmp_global.g.g_abort);
}
}
TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
KMP_MB();
return arg;
}
#endif
void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
kmp_thread_t handle;
DWORD idThread;
KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
th->th.th_info.ds.ds_gtid = gtid;
if (KMP_UBER_GTID(gtid)) {
int stack_data;
/* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
other threads to use. Is it appropriate to just use GetCurrentThread?
When should we close this handle? When unregistering the root? */
{
BOOL rc;
rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
FALSE, DUPLICATE_SAME_ACCESS);
KMP_ASSERT(rc);
KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
"handle = %" KMP_UINTPTR_SPEC "\n",
(LPVOID)th, th->th.th_info.ds.ds_thread));
th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
}
if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
/* we will dynamically update the stack range if gtid_mode == 1 */
TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
__kmp_check_stack_overlap(th);
}
} else {
KMP_MB(); /* Flush all pending memory write invalidates. */
/* Set stack size for this thread now. */
KA_TRACE(10,
("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
stack_size));
stack_size += gtid * __kmp_stkoffset;
TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
KA_TRACE(10,
("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
" bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
(SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
(LPVOID)th, &idThread));
handle = CreateThread(
NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
(LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
KA_TRACE(10,
("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
" bytes, &__kmp_launch_worker = %p, th = %p, "
"idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
(SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
(LPVOID)th, idThread, handle));
if (handle == 0) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
} else {
th->th.th_info.ds.ds_thread = handle;
}
KMP_MB(); /* Flush all pending memory write invalidates. */
}
KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
}
int __kmp_still_running(kmp_info_t *th) {
return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
}
#if KMP_USE_MONITOR
void __kmp_create_monitor(kmp_info_t *th) {
kmp_thread_t handle;
DWORD idThread;
int ideal, new_ideal;
if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
// We don't need monitor thread in case of MAX_BLOCKTIME
KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
"MAX blocktime\n"));
th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
th->th.th_info.ds.ds_gtid = 0;
TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
return;
}
KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
KMP_MB(); /* Flush all pending memory write invalidates. */
__kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
if (__kmp_monitor_ev == NULL) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
}
#if USE_ITT_BUILD
__kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
#endif /* USE_ITT_BUILD */
th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
// FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
// to automatically expand stacksize based on CreateThread error code.
if (__kmp_monitor_stksize == 0) {
__kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
}
if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
__kmp_monitor_stksize = __kmp_sys_min_stksize;
}
KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
(int)__kmp_monitor_stksize));
TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
handle =
CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
(LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
if (handle == 0) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
} else
th->th.th_info.ds.ds_thread = handle;
KMP_MB(); /* Flush all pending memory write invalidates. */
KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
(void *)th->th.th_info.ds.ds_thread));
}
#endif
/* Check to see if thread is still alive.
NOTE: The ExitProcess(code) system call causes all threads to Terminate
with a exit_val = code. Because of this we can not rely on exit_val having
any particular value. So this routine may return STILL_ALIVE in exit_val
even after the thread is dead. */
int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
DWORD rc;
rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
if (rc == 0) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
__kmp_msg_null);
}
return (*exit_val == STILL_ACTIVE);
}
void __kmp_exit_thread(int exit_status) {
ExitThread(exit_status);
} // __kmp_exit_thread
// This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
static void __kmp_reap_common(kmp_info_t *th) {
DWORD exit_val;
KMP_MB(); /* Flush all pending memory write invalidates. */
KA_TRACE(
10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
/* 2006-10-19:
There are two opposite situations:
1. Windows* OS keep thread alive after it resets ds_alive flag and
exits from thread function. (For example, see C70770/Q394281 "unloading of
dll based on OMP is very slow".)
2. Windows* OS may kill thread before it resets ds_alive flag.
Right solution seems to be waiting for *either* thread termination *or*
ds_alive resetting. */
{
// TODO: This code is very similar to KMP_WAIT. Need to generalize
// KMP_WAIT to cover this usage also.
void *obj = NULL;
kmp_uint32 spins;
kmp_uint64 time;
#if USE_ITT_BUILD
KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
#endif /* USE_ITT_BUILD */
KMP_INIT_YIELD(spins);
KMP_INIT_BACKOFF(time);
do {
#if USE_ITT_BUILD
KMP_FSYNC_SPIN_PREPARE(obj);
#endif /* USE_ITT_BUILD */
__kmp_is_thread_alive(th, &exit_val);
KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
} while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
#if USE_ITT_BUILD
if (exit_val == STILL_ACTIVE) {
KMP_FSYNC_CANCEL(obj);
} else {
KMP_FSYNC_SPIN_ACQUIRED(obj);
}
#endif /* USE_ITT_BUILD */
}
__kmp_free_handle(th->th.th_info.ds.ds_thread);
/* NOTE: The ExitProcess(code) system call causes all threads to Terminate
with a exit_val = code. Because of this we can not rely on exit_val having
any particular value. */
kmp_intptr_t e = (kmp_intptr_t)exit_val;
if (exit_val == STILL_ACTIVE) {
KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
} else if ((void *)e != (void *)th) {
KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
}
KA_TRACE(10,
("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
"\n",
th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
th->th.th_info.ds.ds_thread = 0;
th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
th->th.th_info.ds.ds_thread_id = 0;
KMP_MB(); /* Flush all pending memory write invalidates. */
}
#if KMP_USE_MONITOR
void __kmp_reap_monitor(kmp_info_t *th) {
int status;
KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
(void *)th->th.th_info.ds.ds_thread));
// If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
// If both tid and gtid are 0, it means the monitor did not ever start.
// If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
return;
}
KMP_MB(); /* Flush all pending memory write invalidates. */
status = SetEvent(__kmp_monitor_ev);
if (status == FALSE) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
}
KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
th->th.th_info.ds.ds_gtid));
__kmp_reap_common(th);
__kmp_free_handle(__kmp_monitor_ev);
KMP_MB(); /* Flush all pending memory write invalidates. */
}
#endif
void __kmp_reap_worker(kmp_info_t *th) {
KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
th->th.th_info.ds.ds_gtid));
__kmp_reap_common(th);
}
#if KMP_HANDLE_SIGNALS
static void __kmp_team_handler(int signo) {
if (__kmp_global.g.g_abort == 0) {
// Stage 1 signal handler, let's shut down all of the threads.
if (__kmp_debug_buf) {
__kmp_dump_debug_buffer();
}
KMP_MB(); // Flush all pending memory write invalidates.
TCW_4(__kmp_global.g.g_abort, signo);
KMP_MB(); // Flush all pending memory write invalidates.
TCW_4(__kmp_global.g.g_done, TRUE);
KMP_MB(); // Flush all pending memory write invalidates.
}
} // __kmp_team_handler
static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
sig_func_t old = signal(signum, handler);
if (old == SIG_ERR) {
int error = errno;
__kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
__kmp_msg_null);
}
return old;
}
static void __kmp_install_one_handler(int sig, sig_func_t handler,
int parallel_init) {
sig_func_t old;
KMP_MB(); /* Flush all pending memory write invalidates. */
KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
if (parallel_init) {
old = __kmp_signal(sig, handler);
// SIG_DFL on Windows* OS in NULL or 0.
if (old == __kmp_sighldrs[sig]) {
__kmp_siginstalled[sig] = 1;
} else { // Restore/keep user's handler if one previously installed.
old = __kmp_signal(sig, old);
}
} else {
// Save initial/system signal handlers to see if user handlers installed.
// 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
// called once with parallel_init == TRUE.
old = __kmp_signal(sig, SIG_DFL);
__kmp_sighldrs[sig] = old;
__kmp_signal(sig, old);
}
KMP_MB(); /* Flush all pending memory write invalidates. */
} // __kmp_install_one_handler
static void __kmp_remove_one_handler(int sig) {
if (__kmp_siginstalled[sig]) {
sig_func_t old;
KMP_MB(); // Flush all pending memory write invalidates.
KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
old = __kmp_signal(sig, __kmp_sighldrs[sig]);
if (old != __kmp_team_handler) {
KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
"restoring: sig=%d\n",
sig));
old = __kmp_signal(sig, old);
}
__kmp_sighldrs[sig] = NULL;
__kmp_siginstalled[sig] = 0;
KMP_MB(); // Flush all pending memory write invalidates.
}
} // __kmp_remove_one_handler
void __kmp_install_signals(int parallel_init) {
KB_TRACE(10, ("__kmp_install_signals: called\n"));
if (!__kmp_handle_signals) {
KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
"handlers not installed\n"));
return;
}
__kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
__kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
__kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
__kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
__kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
__kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
} // __kmp_install_signals
void __kmp_remove_signals(void) {
int sig;
KB_TRACE(10, ("__kmp_remove_signals: called\n"));
for (sig = 1; sig < NSIG; ++sig) {
__kmp_remove_one_handler(sig);
}
} // __kmp_remove_signals
#endif // KMP_HANDLE_SIGNALS
/* Put the thread to sleep for a time period */
void __kmp_thread_sleep(int millis) {
DWORD status;
status = SleepEx((DWORD)millis, FALSE);
if (status) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
__kmp_msg_null);
}
}
// Determine whether the given address is mapped into the current address space.
int __kmp_is_address_mapped(void *addr) {
MEMORY_BASIC_INFORMATION lpBuffer;
SIZE_T dwLength;
dwLength = sizeof(MEMORY_BASIC_INFORMATION);
VirtualQuery(addr, &lpBuffer, dwLength);
return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
((lpBuffer.Protect == PAGE_NOACCESS) ||
(lpBuffer.Protect == PAGE_EXECUTE)));
}
kmp_uint64 __kmp_hardware_timestamp(void) {
kmp_uint64 r = 0;
QueryPerformanceCounter((LARGE_INTEGER *)&r);
return r;
}
/* Free handle and check the error code */
void __kmp_free_handle(kmp_thread_t tHandle) {
/* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
* as HANDLE */
BOOL rc;
rc = CloseHandle(tHandle);
if (!rc) {
DWORD error = GetLastError();
__kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
}
}
int __kmp_get_load_balance(int max) {
static ULONG glb_buff_size = 100 * 1024;
// Saved count of the running threads for the thread balance algorithm
static int glb_running_threads = 0;
static double glb_call_time = 0; /* Thread balance algorithm call time */
int running_threads = 0; // Number of running threads in the system.
NTSTATUS status = 0;
ULONG buff_size = 0;
ULONG info_size = 0;
void *buffer = NULL;
PSYSTEM_PROCESS_INFORMATION spi = NULL;
int first_time = 1;
double call_time = 0.0; // start, finish;
__kmp_elapsed(&call_time);
if (glb_call_time &&
(call_time - glb_call_time < __kmp_load_balance_interval)) {
running_threads = glb_running_threads;
goto finish;
}
glb_call_time = call_time;
// Do not spend time on running algorithm if we have a permanent error.
if (NtQuerySystemInformation == NULL) {
running_threads = -1;
goto finish;
}
if (max <= 0) {
max = INT_MAX;
}
do {
if (first_time) {
buff_size = glb_buff_size;
} else {
buff_size = 2 * buff_size;
}
buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
if (buffer == NULL) {
running_threads = -1;
goto finish;
}
status = NtQuerySystemInformation(SystemProcessInformation, buffer,
buff_size, &info_size);
first_time = 0;
} while (status == STATUS_INFO_LENGTH_MISMATCH);
glb_buff_size = buff_size;
#define CHECK(cond) \
{ \
KMP_DEBUG_ASSERT(cond); \
if (!(cond)) { \
running_threads = -1; \
goto finish; \
} \
}
CHECK(buff_size >= info_size);
spi = PSYSTEM_PROCESS_INFORMATION(buffer);
for (;;) {
ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
CHECK(0 <= offset &&
offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
HANDLE pid = spi->ProcessId;
ULONG num = spi->NumberOfThreads;
CHECK(num >= 1);
size_t spi_size =
sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
CHECK(offset + spi_size <
info_size); // Make sure process info record fits the buffer.
if (spi->NextEntryOffset != 0) {
CHECK(spi_size <=
spi->NextEntryOffset); // And do not overlap with the next record.
}
// pid == 0 corresponds to the System Idle Process. It always has running
// threads on all cores. So, we don't consider the running threads of this
// process.
if (pid != 0) {
for (ULONG i = 0; i < num; ++i) {
THREAD_STATE state = spi->Threads[i].State;
// Count threads that have Ready or Running state.
// !!! TODO: Why comment does not match the code???
if (state == StateRunning) {
++running_threads;
// Stop counting running threads if the number is already greater than
// the number of available cores
if (running_threads >= max) {
goto finish;
}
}
}
}
if (spi->NextEntryOffset == 0) {
break;
}
spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
}
#undef CHECK
finish: // Clean up and exit.
if (buffer != NULL) {
KMP_INTERNAL_FREE(buffer);
}
glb_running_threads = running_threads;
return running_threads;
} //__kmp_get_load_balance()
// Find symbol from the loaded modules
void *__kmp_lookup_symbol(const char *name, bool next) {
HANDLE process = GetCurrentProcess();
DWORD needed;
HMODULE *modules = nullptr;
if (!EnumProcessModules(process, modules, 0, &needed))
return nullptr;
DWORD num_modules = needed / sizeof(HMODULE);
modules = (HMODULE *)malloc(num_modules * sizeof(HMODULE));
if (!EnumProcessModules(process, modules, needed, &needed)) {
free(modules);
return nullptr;
}
HMODULE curr_module = nullptr;
if (next) {
// Current module needs to be skipped if next flag is true
if (!GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
(LPCTSTR)&__kmp_lookup_symbol, &curr_module)) {
free(modules);
return nullptr;
}
}
void *proc = nullptr;
for (uint32_t i = 0; i < num_modules; i++) {
if (next && modules[i] == curr_module)
continue;
proc = (void *)GetProcAddress(modules[i], name);
if (proc)
break;
}
free(modules);
return proc;
}
// Functions for hidden helper task
void __kmp_hidden_helper_worker_thread_wait() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_do_initialize_hidden_helper_threads() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_threads_initz_wait() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_initz_release() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_main_thread_wait() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_main_thread_release() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_worker_thread_signal() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_threads_deinitz_wait() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
void __kmp_hidden_helper_threads_deinitz_release() {
KMP_ASSERT(0 && "Hidden helper task is not supported on Windows");
}
Codebase Browser
llvm