17 #include "kmp_affinity.h" 21 #include "kmp_wait_release.h" 30 enum SYSTEM_INFORMATION_CLASS {
31 SystemProcessInformation = 5
51 SIZE_T PeakVirtualSize;
54 SIZE_T PeakWorkingSetSize;
55 SIZE_T WorkingSetSize;
56 SIZE_T QuotaPeakPagedPoolUsage;
57 SIZE_T QuotaPagedPoolUsage;
58 SIZE_T QuotaPeakNonPagedPoolUsage;
59 SIZE_T QuotaNonPagedPoolUsage;
61 SIZE_T PeakPagefileUsage;
62 SIZE_T PrivatePageCount;
65 struct SYSTEM_THREAD {
66 LARGE_INTEGER KernelTime;
67 LARGE_INTEGER UserTime;
68 LARGE_INTEGER CreateTime;
74 ULONG ContextSwitchCount;
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
84 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
85 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
88 struct SYSTEM_PROCESS_INFORMATION {
89 ULONG NextEntryOffset;
90 ULONG NumberOfThreads;
91 LARGE_INTEGER Reserved[3];
92 LARGE_INTEGER CreateTime;
93 LARGE_INTEGER UserTime;
94 LARGE_INTEGER KernelTime;
95 UNICODE_STRING ImageName;
98 HANDLE ParentProcessId;
101 VM_COUNTERS VMCounters;
102 IO_COUNTERS IOCounters;
103 SYSTEM_THREAD Threads[1];
105 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
107 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
113 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
119 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
120 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
121 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
124 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
125 PVOID, ULONG, PULONG);
126 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
128 HMODULE ntdll = NULL;
132 static HMODULE kernel32 = NULL;
134 #if KMP_HANDLE_SIGNALS 135 typedef void (*sig_func_t)(int);
136 static sig_func_t __kmp_sighldrs[NSIG];
137 static int __kmp_siginstalled[NSIG];
141 static HANDLE __kmp_monitor_ev;
143 static kmp_int64 __kmp_win32_time;
144 double __kmp_win32_tick;
146 int __kmp_init_runtime = FALSE;
147 CRITICAL_SECTION __kmp_win32_section;
149 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
150 InitializeCriticalSection(&mx->cs);
152 __kmp_itt_system_object_created(&mx->cs,
"Critical Section");
156 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
157 DeleteCriticalSection(&mx->cs);
160 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
161 EnterCriticalSection(&mx->cs);
164 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
165 LeaveCriticalSection(&mx->cs);
168 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
169 cv->waiters_count_ = 0;
170 cv->wait_generation_count_ = 0;
171 cv->release_count_ = 0;
174 __kmp_win32_mutex_init(&cv->waiters_count_lock_);
177 cv->event_ = CreateEvent(NULL,
182 __kmp_itt_system_object_created(cv->event_,
"Event");
186 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
187 __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
188 __kmp_free_handle(cv->event_);
189 memset(cv,
'\0',
sizeof(*cv));
195 void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
196 kmp_info_t *th,
int need_decrease_load) {
201 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
204 cv->waiters_count_++;
207 my_generation = cv->wait_generation_count_;
209 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
210 __kmp_win32_mutex_unlock(mx);
216 WaitForSingleObject(cv->event_, INFINITE);
218 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
223 wait_done = (cv->release_count_ > 0) &&
224 (cv->wait_generation_count_ != my_generation);
226 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
234 __kmp_win32_mutex_lock(mx);
235 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
237 cv->waiters_count_--;
238 cv->release_count_--;
240 last_waiter = (cv->release_count_ == 0);
242 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
246 ResetEvent(cv->event_);
250 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
251 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
253 if (cv->waiters_count_ > 0) {
254 SetEvent(cv->event_);
257 cv->release_count_ = cv->waiters_count_;
260 cv->wait_generation_count_++;
263 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
266 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
267 __kmp_win32_cond_broadcast(cv);
270 void __kmp_enable(
int new_state) {
271 if (__kmp_init_runtime)
272 LeaveCriticalSection(&__kmp_win32_section);
275 void __kmp_disable(
int *old_state) {
278 if (__kmp_init_runtime)
279 EnterCriticalSection(&__kmp_win32_section);
282 void __kmp_suspend_initialize(
void) {
285 static void __kmp_suspend_initialize_thread(kmp_info_t *th) {
286 if (!TCR_4(th->th.th_suspend_init)) {
289 __kmp_win32_cond_init(&th->th.th_suspend_cv);
290 __kmp_win32_mutex_init(&th->th.th_suspend_mx);
291 TCW_4(th->th.th_suspend_init, TRUE);
295 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
296 if (TCR_4(th->th.th_suspend_init)) {
299 __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
300 __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
301 TCW_4(th->th.th_suspend_init, FALSE);
308 static inline void __kmp_suspend_template(
int th_gtid, C *flag) {
309 kmp_info_t *th = __kmp_threads[th_gtid];
311 typename C::flag_t old_spin;
313 KF_TRACE(30, (
"__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
314 th_gtid, flag->get()));
316 __kmp_suspend_initialize_thread(th);
317 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
319 KF_TRACE(10, (
"__kmp_suspend_template: T#%d setting sleep bit for flag's" 321 th_gtid, flag->get()));
325 old_spin = flag->set_sleeping();
327 KF_TRACE(5, (
"__kmp_suspend_template: T#%d set sleep bit for flag's" 329 th_gtid, flag->get(), *(flag->get())));
331 if (flag->done_check_val(old_spin)) {
332 old_spin = flag->unset_sleeping();
333 KF_TRACE(5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit " 334 "for flag's loc(%p)\n",
335 th_gtid, flag->get()));
338 __kmp_suspend_count++;
343 int deactivated = FALSE;
344 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
345 while (flag->is_sleeping()) {
346 KF_TRACE(15, (
"__kmp_suspend_template: T#%d about to perform " 347 "kmp_win32_cond_wait()\n",
352 th->th.th_active = FALSE;
353 if (th->th.th_active_in_pool) {
354 th->th.th_active_in_pool = FALSE;
355 KMP_TEST_THEN_DEC32((kmp_int32 *)&__kmp_thread_pool_active_nth);
356 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
360 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
363 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
368 if (flag->is_sleeping()) {
370 (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
378 th->th.th_active = TRUE;
379 if (TCR_4(th->th.th_in_pool)) {
380 KMP_TEST_THEN_INC32((kmp_int32 *)&__kmp_thread_pool_active_nth);
381 th->th.th_active_in_pool = TRUE;
386 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
388 KF_TRACE(30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid));
391 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
392 __kmp_suspend_template(th_gtid, flag);
394 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
395 __kmp_suspend_template(th_gtid, flag);
397 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
398 __kmp_suspend_template(th_gtid, flag);
404 static inline void __kmp_resume_template(
int target_gtid, C *flag) {
405 kmp_info_t *th = __kmp_threads[target_gtid];
409 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
412 KF_TRACE(30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
415 __kmp_suspend_initialize_thread(th);
416 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
419 flag = (C *)th->th.th_sleep_loc;
424 if (!flag || flag->get_type() != flag->get_ptr_type()) {
427 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already " 428 "awake: flag's loc(%p)\n",
429 gtid, target_gtid, NULL));
430 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
433 typename C::flag_t old_spin = flag->unset_sleeping();
434 if (!flag->is_sleeping_val(old_spin)) {
435 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already " 436 "awake: flag's loc(%p): %u => %u\n",
437 gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
438 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
442 TCW_PTR(th->th.th_sleep_loc, NULL);
443 KF_TRACE(5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep " 444 "bit for flag's loc(%p)\n",
445 gtid, target_gtid, flag->get()));
447 __kmp_win32_cond_signal(&th->th.th_suspend_cv);
448 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
450 KF_TRACE(30, (
"__kmp_resume_template: T#%d exiting after signaling wake up" 455 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
456 __kmp_resume_template(target_gtid, flag);
458 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
459 __kmp_resume_template(target_gtid, flag);
461 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
462 __kmp_resume_template(target_gtid, flag);
465 void __kmp_yield(
int cond) {
470 void __kmp_gtid_set_specific(
int gtid) {
471 if (__kmp_init_gtid) {
472 KA_TRACE(50, (
"__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
473 __kmp_gtid_threadprivate_key));
474 if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
475 KMP_FATAL(TLSSetValueFailed);
477 KA_TRACE(50, (
"__kmp_gtid_set_specific: runtime shutdown, returning\n"));
481 int __kmp_gtid_get_specific() {
483 if (!__kmp_init_gtid) {
484 KA_TRACE(50, (
"__kmp_gtid_get_specific: runtime shutdown, returning " 485 "KMP_GTID_SHUTDOWN\n"));
486 return KMP_GTID_SHUTDOWN;
488 gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
494 KA_TRACE(50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
495 __kmp_gtid_threadprivate_key, gtid));
499 void __kmp_affinity_bind_thread(
int proc) {
500 if (__kmp_num_proc_groups > 1) {
504 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
505 sizeof(DWORD_PTR))));
506 ga.Group = proc / (CHAR_BIT *
sizeof(DWORD_PTR));
507 ga.Mask = (
unsigned long long)1 << (proc % (CHAR_BIT *
sizeof(DWORD_PTR)));
508 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
510 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
511 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
512 DWORD error = GetLastError();
513 if (__kmp_affinity_verbose) {
514 kmp_msg_t err_code = KMP_ERR(error);
515 __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
517 if (__kmp_generate_warnings == kmp_warnings_off) {
518 __kmp_str_free(&err_code.str);
523 kmp_affin_mask_t *mask;
524 KMP_CPU_ALLOC_ON_STACK(mask);
526 KMP_CPU_SET(proc, mask);
527 __kmp_set_system_affinity(mask, TRUE);
528 KMP_CPU_FREE_FROM_STACK(mask);
532 void __kmp_affinity_determine_capable(
const char *env_var) {
535 #if KMP_GROUP_AFFINITY 536 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups *
sizeof(DWORD_PTR));
538 KMP_AFFINITY_ENABLE(
sizeof(DWORD_PTR));
541 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 542 "Windows* OS affinity interface functional (mask size = " 543 "%" KMP_SIZE_T_SPEC
").\n",
544 __kmp_affin_mask_size));
547 double __kmp_read_cpu_time(
void) {
548 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
554 status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
555 &KernelTime, &UserTime);
560 sec += KernelTime.dwHighDateTime;
561 sec += UserTime.dwHighDateTime;
564 sec *= (double)(1 << 16) * (double)(1 << 16);
566 sec += KernelTime.dwLowDateTime;
567 sec += UserTime.dwLowDateTime;
569 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
575 int __kmp_read_system_info(
struct kmp_sys_info *info) {
588 void __kmp_runtime_initialize(
void) {
593 if (__kmp_init_runtime) {
601 UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
603 BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
604 GET_MODULE_HANDLE_EX_FLAG_PIN,
605 (LPCTSTR)&__kmp_serial_initialize, &h);
606 KMP_DEBUG_ASSERT2(h && ret,
"OpenMP RTL cannot find itself loaded");
607 SetErrorMode(err_mode);
608 KA_TRACE(10, (
"__kmp_runtime_initialize: dynamic library pinned\n"));
612 InitializeCriticalSection(&__kmp_win32_section);
614 __kmp_itt_system_object_created(&__kmp_win32_section,
"Critical Section");
616 __kmp_initialize_system_tick();
618 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 619 if (!__kmp_cpuinfo.initialized) {
620 __kmp_query_cpuid(&__kmp_cpuinfo);
625 #if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB 640 __kmp_tls_gtid_min = 0;
642 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
646 if (!__kmp_gtid_threadprivate_key) {
647 __kmp_gtid_threadprivate_key = TlsAlloc();
648 if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
649 KMP_FATAL(TLSOutOfIndexes);
657 __kmp_str_buf_init(&path);
658 path_size = GetSystemDirectory(path.str, path.size);
659 KMP_DEBUG_ASSERT(path_size > 0);
660 if (path_size >= path.size) {
662 __kmp_str_buf_reserve(&path, path_size);
663 path_size = GetSystemDirectory(path.str, path.size);
664 KMP_DEBUG_ASSERT(path_size > 0);
666 if (path_size > 0 && path_size < path.size) {
669 path.used = path_size;
670 __kmp_str_buf_print(&path,
"\\%s",
"ntdll.dll");
673 ntdll = GetModuleHandle(path.str);
676 KMP_DEBUG_ASSERT(ntdll != NULL);
678 NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
679 ntdll,
"NtQuerySystemInformation");
681 KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
683 #if KMP_GROUP_AFFINITY 686 if (path_size > 0 && path_size < path.size) {
689 path.used = path_size;
690 __kmp_str_buf_print(&path,
"\\%s",
"kernel32.dll");
693 kernel32 = GetModuleHandle(path.str);
694 KA_TRACE(10, (
"__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
698 if (kernel32 != NULL) {
699 __kmp_GetActiveProcessorCount =
700 (kmp_GetActiveProcessorCount_t)GetProcAddress(
701 kernel32,
"GetActiveProcessorCount");
702 __kmp_GetActiveProcessorGroupCount =
703 (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
704 kernel32,
"GetActiveProcessorGroupCount");
705 __kmp_GetThreadGroupAffinity =
706 (kmp_GetThreadGroupAffinity_t)GetProcAddress(
707 kernel32,
"GetThreadGroupAffinity");
708 __kmp_SetThreadGroupAffinity =
709 (kmp_SetThreadGroupAffinity_t)GetProcAddress(
710 kernel32,
"SetThreadGroupAffinity");
712 KA_TRACE(10, (
"__kmp_runtime_initialize: __kmp_GetActiveProcessorCount" 714 __kmp_GetActiveProcessorCount));
715 KA_TRACE(10, (
"__kmp_runtime_initialize: " 716 "__kmp_GetActiveProcessorGroupCount = %p\n",
717 __kmp_GetActiveProcessorGroupCount));
718 KA_TRACE(10, (
"__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity" 720 __kmp_GetThreadGroupAffinity));
721 KA_TRACE(10, (
"__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity" 723 __kmp_SetThreadGroupAffinity));
724 KA_TRACE(10, (
"__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
725 sizeof(kmp_affin_mask_t)));
732 if ((__kmp_GetActiveProcessorCount != NULL) &&
733 (__kmp_GetActiveProcessorGroupCount != NULL) &&
734 (__kmp_GetThreadGroupAffinity != NULL) &&
735 (__kmp_SetThreadGroupAffinity != NULL) &&
736 ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
741 KA_TRACE(10, (
"__kmp_runtime_initialize: %d processor groups" 743 __kmp_num_proc_groups));
747 for (i = 0; i < __kmp_num_proc_groups; i++) {
748 DWORD size = __kmp_GetActiveProcessorCount(i);
750 KA_TRACE(10, (
"__kmp_runtime_initialize: proc group %d size = %d\n",
754 KA_TRACE(10, (
"__kmp_runtime_initialize: %d processor groups" 756 __kmp_num_proc_groups));
760 if (__kmp_num_proc_groups <= 1) {
761 GetSystemInfo(&info);
762 __kmp_xproc = info.dwNumberOfProcessors;
765 GetSystemInfo(&info);
766 __kmp_xproc = info.dwNumberOfProcessors;
771 if (__kmp_xproc <= 0) {
776 (
"__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
778 __kmp_str_buf_free(&path);
781 __kmp_itt_initialize();
784 __kmp_init_runtime = TRUE;
787 void __kmp_runtime_destroy(
void) {
788 if (!__kmp_init_runtime) {
798 KA_TRACE(40, (
"__kmp_runtime_destroy\n"));
800 if (__kmp_gtid_threadprivate_key) {
801 TlsFree(__kmp_gtid_threadprivate_key);
802 __kmp_gtid_threadprivate_key = 0;
805 __kmp_affinity_uninitialize();
806 DeleteCriticalSection(&__kmp_win32_section);
809 NtQuerySystemInformation = NULL;
813 __kmp_GetActiveProcessorCount = NULL;
814 __kmp_GetActiveProcessorGroupCount = NULL;
815 __kmp_GetThreadGroupAffinity = NULL;
816 __kmp_SetThreadGroupAffinity = NULL;
817 #endif // KMP_ARCH_X86_64 819 __kmp_init_runtime = FALSE;
822 void __kmp_terminate_thread(
int gtid) {
823 kmp_info_t *th = __kmp_threads[gtid];
828 KA_TRACE(10, (
"__kmp_terminate_thread: kill (%d)\n", gtid));
830 if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
833 __kmp_free_handle(th->th.th_info.ds.ds_thread);
836 void __kmp_clear_system_time(
void) {
839 status = QueryPerformanceCounter(&time);
840 __kmp_win32_time = (kmp_int64)time.QuadPart;
843 void __kmp_initialize_system_tick(
void) {
848 status = QueryPerformanceFrequency(&freq);
850 DWORD error = GetLastError();
851 __kmp_msg(kmp_ms_fatal,
852 KMP_MSG(FunctionError,
"QueryPerformanceFrequency()"),
853 KMP_ERR(error), __kmp_msg_null);
856 __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
863 void __kmp_elapsed(
double *t) {
866 status = QueryPerformanceCounter(&now);
867 *t = ((double)now.QuadPart) * __kmp_win32_tick;
872 void __kmp_elapsed_tick(
double *t) { *t = __kmp_win32_tick; }
874 void __kmp_read_system_time(
double *delta) {
879 status = QueryPerformanceCounter(&now);
881 *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
887 kmp_uint64 __kmp_now_nsec() {
889 QueryPerformanceCounter(&now);
890 return 1e9 * __kmp_win32_tick * now.QuadPart;
893 void *__stdcall __kmp_launch_worker(
void *arg) {
894 volatile void *stack_data;
897 kmp_info_t *this_thr = (kmp_info_t *)arg;
900 gtid = this_thr->th.th_info.ds.ds_gtid;
901 __kmp_gtid_set_specific(gtid);
902 #ifdef KMP_TDATA_GTID 903 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 904 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 905 "reference: http://support.microsoft.com/kb/118816" 910 __kmp_itt_thread_name(gtid);
913 __kmp_affinity_set_init_mask(gtid, FALSE);
915 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 917 __kmp_clear_x87_fpu_status_word();
918 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
919 __kmp_load_mxcsr(&__kmp_init_mxcsr);
922 if (__kmp_stkoffset > 0 && gtid > 0) {
923 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
926 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
927 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
928 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
930 if (TCR_4(__kmp_gtid_mode) <
932 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
933 KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
934 __kmp_check_stack_overlap(this_thr);
937 exit_val = __kmp_launch_thread(this_thr);
938 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
939 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
947 void *__stdcall __kmp_launch_monitor(
void *arg) {
949 kmp_thread_t monitor;
952 kmp_info_t *this_thr = (kmp_info_t *)arg;
954 KMP_DEBUG_ASSERT(__kmp_init_monitor);
955 TCW_4(__kmp_init_monitor, 2);
957 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
958 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
961 KA_TRACE(10, (
"__kmp_launch_monitor: launched\n"));
963 monitor = GetCurrentThread();
966 status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
968 DWORD error = GetLastError();
969 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantSetThreadPriority), KMP_ERR(error),
974 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
975 #ifdef KMP_TDATA_GTID 976 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 977 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 978 "reference: http://support.microsoft.com/kb/118816" 983 __kmp_itt_thread_ignore();
989 interval = (1000 / __kmp_monitor_wakeups);
991 while (!TCR_4(__kmp_global.g.g_done)) {
994 KA_TRACE(15, (
"__kmp_launch_monitor: update\n"));
996 wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
998 if (wait_status == WAIT_TIMEOUT) {
999 TCW_4(__kmp_global.g.g_time.dt.t_value,
1000 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1006 KA_TRACE(10, (
"__kmp_launch_monitor: finished\n"));
1008 status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1010 DWORD error = GetLastError();
1011 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantSetThreadPriority), KMP_ERR(error),
1015 if (__kmp_global.g.g_abort != 0) {
1020 KA_TRACE(10, (
"__kmp_launch_monitor: terminate sig=%d\n",
1021 (__kmp_global.g.g_abort)));
1026 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1027 __kmp_terminate_thread(gtid);
1034 (
"__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1036 if (__kmp_global.g.g_abort > 0) {
1037 raise(__kmp_global.g.g_abort);
1041 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1048 void __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size) {
1049 kmp_thread_t handle;
1052 KA_TRACE(10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid));
1054 th->th.th_info.ds.ds_gtid = gtid;
1056 if (KMP_UBER_GTID(gtid)) {
1064 rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1065 GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1066 FALSE, DUPLICATE_SAME_ACCESS);
1068 KA_TRACE(10, (
" __kmp_create_worker: ROOT Handle duplicated, th = %p, " 1069 "handle = %" KMP_UINTPTR_SPEC
"\n",
1070 (LPVOID)th, th->th.th_info.ds.ds_thread));
1071 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1073 if (TCR_4(__kmp_gtid_mode) < 2) {
1075 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1076 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1077 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1078 __kmp_check_stack_overlap(th);
1085 (
"__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
" bytes\n",
1088 stack_size += gtid * __kmp_stkoffset;
1090 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1091 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1094 (
"__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1095 " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1096 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1097 (LPVOID)th, &idThread));
1099 handle = CreateThread(
1100 NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1101 (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1104 (
"__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1105 " bytes, &__kmp_launch_worker = %p, th = %p, " 1106 "idThread = %u, handle = %" KMP_UINTPTR_SPEC
"\n",
1107 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1108 (LPVOID)th, idThread, handle));
1111 DWORD error = GetLastError();
1112 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCreateThread), KMP_ERR(error),
1115 th->th.th_info.ds.ds_thread = handle;
1121 KA_TRACE(10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid));
1124 int __kmp_still_running(kmp_info_t *th) {
1125 return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1129 void __kmp_create_monitor(kmp_info_t *th) {
1130 kmp_thread_t handle;
1132 int ideal, new_ideal;
1134 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1136 KA_TRACE(10, (
"__kmp_create_monitor: skipping monitor thread because of " 1137 "MAX blocktime\n"));
1138 th->th.th_info.ds.ds_tid = 0;
1139 th->th.th_info.ds.ds_gtid = 0;
1140 TCW_4(__kmp_init_monitor, 2);
1143 KA_TRACE(10, (
"__kmp_create_monitor: try to create monitor\n"));
1147 __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1148 if (__kmp_monitor_ev == NULL) {
1149 DWORD error = GetLastError();
1150 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCreateEvent), KMP_ERR(error),
1154 __kmp_itt_system_object_created(__kmp_monitor_ev,
"Event");
1157 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1158 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1162 if (__kmp_monitor_stksize == 0) {
1163 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1165 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1166 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1169 KA_TRACE(10, (
"__kmp_create_monitor: requested stacksize = %d bytes\n",
1170 (
int)__kmp_monitor_stksize));
1172 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1175 CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1176 (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1177 STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1179 DWORD error = GetLastError();
1180 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCreateThread), KMP_ERR(error),
1183 th->th.th_info.ds.ds_thread = handle;
1187 KA_TRACE(10, (
"__kmp_create_monitor: monitor created %p\n",
1188 (
void *)th->th.th_info.ds.ds_thread));
1198 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1200 rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1202 DWORD error = GetLastError();
1203 __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError,
"GetExitCodeThread()"),
1204 KMP_ERR(error), __kmp_msg_null);
1206 return (*exit_val == STILL_ACTIVE);
1209 void __kmp_exit_thread(
int exit_status) {
1210 ExitThread(exit_status);
1214 static void __kmp_reap_common(kmp_info_t *th) {
1220 10, (
"__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1237 KMP_FSYNC_SPIN_INIT(obj, (
void *)&th->th.th_info.ds.ds_alive);
1239 KMP_INIT_YIELD(spins);
1242 KMP_FSYNC_SPIN_PREPARE(obj);
1244 __kmp_is_thread_alive(th, &exit_val);
1245 KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
1246 KMP_YIELD_SPIN(spins);
1247 }
while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1249 if (exit_val == STILL_ACTIVE) {
1250 KMP_FSYNC_CANCEL(obj);
1252 KMP_FSYNC_SPIN_ACQUIRED(obj);
1257 __kmp_free_handle(th->th.th_info.ds.ds_thread);
1262 if (exit_val == STILL_ACTIVE) {
1263 KA_TRACE(1, (
"__kmp_reap_common: thread still active.\n"));
1264 }
else if ((
void *)exit_val != (
void *)th) {
1265 KA_TRACE(1, (
"__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1269 (
"__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1271 th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1273 th->th.th_info.ds.ds_thread = 0;
1274 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1275 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1276 th->th.th_info.ds.ds_thread_id = 0;
1282 void __kmp_reap_monitor(kmp_info_t *th) {
1285 KA_TRACE(10, (
"__kmp_reap_monitor: try to reap %p\n",
1286 (
void *)th->th.th_info.ds.ds_thread));
1291 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1292 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1293 KA_TRACE(10, (
"__kmp_reap_monitor: monitor did not start, returning\n"));
1299 status = SetEvent(__kmp_monitor_ev);
1300 if (status == FALSE) {
1301 DWORD error = GetLastError();
1302 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantSetEvent), KMP_ERR(error),
1305 KA_TRACE(10, (
"__kmp_reap_monitor: reaping thread (%d)\n",
1306 th->th.th_info.ds.ds_gtid));
1307 __kmp_reap_common(th);
1309 __kmp_free_handle(__kmp_monitor_ev);
1315 void __kmp_reap_worker(kmp_info_t *th) {
1316 KA_TRACE(10, (
"__kmp_reap_worker: reaping thread (%d)\n",
1317 th->th.th_info.ds.ds_gtid));
1318 __kmp_reap_common(th);
1321 #if KMP_HANDLE_SIGNALS 1323 static void __kmp_team_handler(
int signo) {
1324 if (__kmp_global.g.g_abort == 0) {
1326 if (__kmp_debug_buf) {
1327 __kmp_dump_debug_buffer();
1330 TCW_4(__kmp_global.g.g_abort, signo);
1332 TCW_4(__kmp_global.g.g_done, TRUE);
1337 static sig_func_t __kmp_signal(
int signum, sig_func_t handler) {
1338 sig_func_t old = signal(signum, handler);
1339 if (old == SIG_ERR) {
1341 __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError,
"signal"), KMP_ERR(error),
1347 static void __kmp_install_one_handler(
int sig, sig_func_t handler,
1348 int parallel_init) {
1351 KB_TRACE(60, (
"__kmp_install_one_handler: called: sig=%d\n", sig));
1352 if (parallel_init) {
1353 old = __kmp_signal(sig, handler);
1355 if (old == __kmp_sighldrs[sig]) {
1356 __kmp_siginstalled[sig] = 1;
1358 old = __kmp_signal(sig, old);
1364 old = __kmp_signal(sig, SIG_DFL);
1365 __kmp_sighldrs[sig] = old;
1366 __kmp_signal(sig, old);
1371 static void __kmp_remove_one_handler(
int sig) {
1372 if (__kmp_siginstalled[sig]) {
1375 KB_TRACE(60, (
"__kmp_remove_one_handler: called: sig=%d\n", sig));
1376 old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1377 if (old != __kmp_team_handler) {
1378 KB_TRACE(10, (
"__kmp_remove_one_handler: oops, not our handler, " 1379 "restoring: sig=%d\n",
1381 old = __kmp_signal(sig, old);
1383 __kmp_sighldrs[sig] = NULL;
1384 __kmp_siginstalled[sig] = 0;
1389 void __kmp_install_signals(
int parallel_init) {
1390 KB_TRACE(10, (
"__kmp_install_signals: called\n"));
1391 if (!__kmp_handle_signals) {
1392 KB_TRACE(10, (
"__kmp_install_signals: KMP_HANDLE_SIGNALS is false - " 1393 "handlers not installed\n"));
1396 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1397 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1398 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1399 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1400 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1401 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1404 void __kmp_remove_signals(
void) {
1406 KB_TRACE(10, (
"__kmp_remove_signals: called\n"));
1407 for (sig = 1; sig < NSIG; ++sig) {
1408 __kmp_remove_one_handler(sig);
1412 #endif // KMP_HANDLE_SIGNALS 1415 void __kmp_thread_sleep(
int millis) {
1418 status = SleepEx((DWORD)millis, FALSE);
1420 DWORD error = GetLastError();
1421 __kmp_msg(kmp_ms_fatal, KMP_MSG(FunctionError,
"SleepEx()"), KMP_ERR(error),
1427 int __kmp_is_address_mapped(
void *addr) {
1429 MEMORY_BASIC_INFORMATION lpBuffer;
1432 dwLength =
sizeof(MEMORY_BASIC_INFORMATION);
1434 status = VirtualQuery(addr, &lpBuffer, dwLength);
1436 return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1437 ((lpBuffer.Protect == PAGE_NOACCESS) ||
1438 (lpBuffer.Protect == PAGE_EXECUTE)));
1441 kmp_uint64 __kmp_hardware_timestamp(
void) {
1444 QueryPerformanceCounter((LARGE_INTEGER *)&r);
1449 void __kmp_free_handle(kmp_thread_t tHandle) {
1453 rc = CloseHandle(tHandle);
1455 DWORD error = GetLastError();
1456 __kmp_msg(kmp_ms_fatal, KMP_MSG(CantCloseHandle), KMP_ERR(error),
1461 int __kmp_get_load_balance(
int max) {
1462 static ULONG glb_buff_size = 100 * 1024;
1465 static int glb_running_threads = 0;
1466 static double glb_call_time = 0;
1468 int running_threads = 0;
1469 NTSTATUS status = 0;
1470 ULONG buff_size = 0;
1471 ULONG info_size = 0;
1472 void *buffer = NULL;
1473 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1476 double call_time = 0.0;
1478 __kmp_elapsed(&call_time);
1480 if (glb_call_time &&
1481 (call_time - glb_call_time < __kmp_load_balance_interval)) {
1482 running_threads = glb_running_threads;
1485 glb_call_time = call_time;
1488 if (NtQuerySystemInformation == NULL) {
1489 running_threads = -1;
1500 buff_size = glb_buff_size;
1502 buff_size = 2 * buff_size;
1505 buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1506 if (buffer == NULL) {
1507 running_threads = -1;
1510 status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1511 buff_size, &info_size);
1514 }
while (status == STATUS_INFO_LENGTH_MISMATCH);
1515 glb_buff_size = buff_size;
1517 #define CHECK(cond) \ 1519 KMP_DEBUG_ASSERT(cond); \ 1521 running_threads = -1; \ 1526 CHECK(buff_size >= info_size);
1527 spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1529 ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1530 CHECK(0 <= offset &&
1531 offset +
sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1532 HANDLE pid = spi->ProcessId;
1533 ULONG num = spi->NumberOfThreads;
1536 sizeof(SYSTEM_PROCESS_INFORMATION) +
sizeof(SYSTEM_THREAD) * (num - 1);
1537 CHECK(offset + spi_size <
1539 if (spi->NextEntryOffset != 0) {
1541 spi->NextEntryOffset);
1547 for (
int i = 0; i < num; ++i) {
1548 THREAD_STATE state = spi->Threads[i].State;
1551 if (state == StateRunning) {
1555 if (running_threads >= max) {
1561 if (spi->NextEntryOffset == 0) {
1564 spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1571 if (buffer != NULL) {
1572 KMP_INTERNAL_FREE(buffer);
1575 glb_running_threads = running_threads;
1577 return running_threads;