20 #include "kmp_wait_release.h" 21 #include "kmp_affinity.h" 30 enum SYSTEM_INFORMATION_CLASS {
31 SystemProcessInformation = 5
51 SIZE_T PeakVirtualSize;
54 SIZE_T PeakWorkingSetSize;
55 SIZE_T WorkingSetSize;
56 SIZE_T QuotaPeakPagedPoolUsage;
57 SIZE_T QuotaPagedPoolUsage;
58 SIZE_T QuotaPeakNonPagedPoolUsage;
59 SIZE_T QuotaNonPagedPoolUsage;
61 SIZE_T PeakPagefileUsage;
62 SIZE_T PrivatePageCount;
65 struct SYSTEM_THREAD {
66 LARGE_INTEGER KernelTime;
67 LARGE_INTEGER UserTime;
68 LARGE_INTEGER CreateTime;
74 ULONG ContextSwitchCount;
79 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
81 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
82 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
84 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
85 KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
88 struct SYSTEM_PROCESS_INFORMATION {
89 ULONG NextEntryOffset;
90 ULONG NumberOfThreads;
91 LARGE_INTEGER Reserved[ 3 ];
92 LARGE_INTEGER CreateTime;
93 LARGE_INTEGER UserTime;
94 LARGE_INTEGER KernelTime;
95 UNICODE_STRING ImageName;
98 HANDLE ParentProcessId;
100 ULONG Reserved2[ 2 ];
101 VM_COUNTERS VMCounters;
102 IO_COUNTERS IOCounters;
103 SYSTEM_THREAD Threads[ 1 ];
105 typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
107 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
108 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
109 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
111 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
112 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
113 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
114 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
115 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
117 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
118 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
119 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
120 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
121 KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
124 typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
125 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
127 HMODULE ntdll = NULL;
131 static HMODULE kernel32 = NULL;
136 #if KMP_HANDLE_SIGNALS 137 typedef void (* sig_func_t )( int );
138 static sig_func_t __kmp_sighldrs[ NSIG ];
139 static int __kmp_siginstalled[ NSIG ];
143 static HANDLE __kmp_monitor_ev;
145 static kmp_int64 __kmp_win32_time;
146 double __kmp_win32_tick;
148 int __kmp_init_runtime = FALSE;
149 CRITICAL_SECTION __kmp_win32_section;
152 __kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
154 InitializeCriticalSection( & mx->cs );
156 __kmp_itt_system_object_created( & mx->cs,
"Critical Section" );
161 __kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
163 DeleteCriticalSection( & mx->cs );
167 __kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
169 EnterCriticalSection( & mx->cs );
173 __kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
175 LeaveCriticalSection( & mx->cs );
179 __kmp_win32_cond_init( kmp_win32_cond_t *cv )
181 cv->waiters_count_ = 0;
182 cv->wait_generation_count_ = 0;
183 cv->release_count_ = 0;
186 __kmp_win32_mutex_init( & cv->waiters_count_lock_ );
189 cv->event_ = CreateEvent( NULL,
194 __kmp_itt_system_object_created( cv->event_,
"Event" );
199 __kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
201 __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
202 __kmp_free_handle( cv->event_ );
203 memset( cv,
'\0',
sizeof( *cv ) );
210 __kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th,
int need_decrease_load )
216 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
219 cv->waiters_count_++;
222 my_generation = cv->wait_generation_count_;
224 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
225 __kmp_win32_mutex_unlock( mx );
231 WaitForSingleObject( cv->event_, INFINITE );
233 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
238 wait_done = ( cv->release_count_ > 0 ) &&
239 ( cv->wait_generation_count_ != my_generation );
241 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
249 __kmp_win32_mutex_lock( mx );
250 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
252 cv->waiters_count_--;
253 cv->release_count_--;
255 last_waiter = ( cv->release_count_ == 0 );
257 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
261 ResetEvent( cv->event_ );
266 __kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
268 __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
270 if( cv->waiters_count_ > 0 ) {
271 SetEvent( cv->event_ );
274 cv->release_count_ = cv->waiters_count_;
277 cv->wait_generation_count_++;
280 __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
284 __kmp_win32_cond_signal( kmp_win32_cond_t *cv )
286 __kmp_win32_cond_broadcast( cv );
293 __kmp_enable(
int new_state )
295 if (__kmp_init_runtime)
296 LeaveCriticalSection( & __kmp_win32_section );
300 __kmp_disable(
int *old_state )
304 if (__kmp_init_runtime)
305 EnterCriticalSection( & __kmp_win32_section );
309 __kmp_suspend_initialize(
void )
315 __kmp_suspend_initialize_thread( kmp_info_t *th )
317 if ( ! TCR_4( th->th.th_suspend_init ) ) {
320 __kmp_win32_cond_init( &th->th.th_suspend_cv );
321 __kmp_win32_mutex_init( &th->th.th_suspend_mx );
322 TCW_4( th->th.th_suspend_init, TRUE );
327 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
329 if ( TCR_4( th->th.th_suspend_init ) ) {
332 __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
333 __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
334 TCW_4( th->th.th_suspend_init, FALSE );
342 static inline void __kmp_suspend_template(
int th_gtid, C *flag )
344 kmp_info_t *th = __kmp_threads[th_gtid];
346 typename C::flag_t old_spin;
348 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) );
350 __kmp_suspend_initialize_thread( th );
351 __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
353 KF_TRACE( 10, (
"__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n",
354 th_gtid, flag->get() ) );
359 old_spin = flag->set_sleeping();
361 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n",
362 th_gtid, flag->get(), *(flag->get()) ) );
364 if ( flag->done_check_val(old_spin) ) {
365 old_spin = flag->unset_sleeping();
366 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n",
367 th_gtid, flag->get()) );
370 __kmp_suspend_count++;
376 int deactivated = FALSE;
377 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
378 while ( flag->is_sleeping() ) {
379 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n",
382 if ( ! deactivated ) {
383 th->th.th_active = FALSE;
384 if ( th->th.th_active_in_pool ) {
385 th->th.th_active_in_pool = FALSE;
387 (kmp_int32 *) &__kmp_thread_pool_active_nth );
388 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
392 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
395 __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
399 if( flag->is_sleeping() ) {
400 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ));
408 th->th.th_active = TRUE;
409 if ( TCR_4(th->th.th_in_pool) ) {
411 (kmp_int32 *) &__kmp_thread_pool_active_nth );
412 th->th.th_active_in_pool = TRUE;
417 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
419 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid ) );
422 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
423 __kmp_suspend_template(th_gtid, flag);
425 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
426 __kmp_suspend_template(th_gtid, flag);
428 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
429 __kmp_suspend_template(th_gtid, flag);
437 static inline void __kmp_resume_template(
int target_gtid, C *flag )
439 kmp_info_t *th = __kmp_threads[target_gtid];
443 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
446 KF_TRACE( 30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
448 __kmp_suspend_initialize_thread( th );
449 __kmp_win32_mutex_lock( &th->th.th_suspend_mx );
452 flag = (C *)th->th.th_sleep_loc;
456 if (!flag || flag->get_type() != flag->get_ptr_type()) {
457 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n",
458 gtid, target_gtid, NULL ) );
459 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
463 typename C::flag_t old_spin = flag->unset_sleeping();
464 if ( !flag->is_sleeping_val(old_spin) ) {
465 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): " 467 gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) );
468 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
472 TCW_PTR(th->th.th_sleep_loc, NULL);
474 KF_TRACE( 5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n",
475 gtid, target_gtid, flag->get() ) );
477 __kmp_win32_cond_signal( &th->th.th_suspend_cv );
478 __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
480 KF_TRACE( 30, (
"__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
481 gtid, target_gtid ) );
484 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
485 __kmp_resume_template(target_gtid, flag);
487 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
488 __kmp_resume_template(target_gtid, flag);
490 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
491 __kmp_resume_template(target_gtid, flag);
499 __kmp_yield(
int cond )
509 __kmp_gtid_set_specific(
int gtid )
511 if( __kmp_init_gtid ) {
512 KA_TRACE( 50, (
"__kmp_gtid_set_specific: T#%d key:%d\n",
513 gtid, __kmp_gtid_threadprivate_key ));
514 if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
515 KMP_FATAL( TLSSetValueFailed );
517 KA_TRACE( 50, (
"__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
522 __kmp_gtid_get_specific()
525 if( !__kmp_init_gtid ) {
526 KA_TRACE( 50, (
"__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
527 return KMP_GTID_SHUTDOWN;
529 gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
536 KA_TRACE( 50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
537 __kmp_gtid_threadprivate_key, gtid ));
545 __kmp_affinity_bind_thread(
int proc )
547 if (__kmp_num_proc_groups > 1) {
553 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
554 * CHAR_BIT *
sizeof(DWORD_PTR))));
555 ga.Group = proc / (CHAR_BIT *
sizeof(DWORD_PTR));
556 ga.Mask = (
unsigned long long)1 << (proc % (CHAR_BIT *
sizeof(DWORD_PTR)));
557 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
559 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
560 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
561 DWORD error = GetLastError();
562 if (__kmp_affinity_verbose) {
563 kmp_msg_t err_code = KMP_ERR( error );
566 KMP_MSG( CantSetThreadAffMask ),
570 if (__kmp_generate_warnings == kmp_warnings_off) {
571 __kmp_str_free(&err_code.str);
576 kmp_affin_mask_t *mask;
577 KMP_CPU_ALLOC_ON_STACK(mask);
579 KMP_CPU_SET(proc, mask);
580 __kmp_set_system_affinity(mask, TRUE);
581 KMP_CPU_FREE_FROM_STACK(mask);
586 __kmp_affinity_determine_capable(
const char *env_var )
592 #if KMP_GROUP_AFFINITY 593 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*
sizeof(DWORD_PTR));
595 KMP_AFFINITY_ENABLE(
sizeof(DWORD_PTR));
599 "__kmp_affinity_determine_capable: " 600 "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC
").\n",
601 __kmp_affin_mask_size
606 __kmp_read_cpu_time(
void )
608 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
614 status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
615 &ExitTime, &KernelTime, &UserTime );
620 sec += KernelTime.dwHighDateTime;
621 sec += UserTime.dwHighDateTime;
624 sec *= (double) (1 << 16) * (double) (1 << 16);
626 sec += KernelTime.dwLowDateTime;
627 sec += UserTime.dwLowDateTime;
629 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
636 __kmp_read_system_info(
struct kmp_sys_info *info )
655 __kmp_runtime_initialize(
void )
661 if ( __kmp_init_runtime ) {
669 UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS);
671 BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS
672 |GET_MODULE_HANDLE_EX_FLAG_PIN,
673 (LPCTSTR)&__kmp_serial_initialize, &h);
674 KMP_DEBUG_ASSERT2(h && ret,
"OpenMP RTL cannot find itself loaded");
675 SetErrorMode (err_mode);
676 KA_TRACE( 10, (
"__kmp_runtime_initialize: dynamic library pinned\n") );
680 InitializeCriticalSection( & __kmp_win32_section );
682 __kmp_itt_system_object_created( & __kmp_win32_section,
"Critical Section" );
684 __kmp_initialize_system_tick();
686 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 687 if ( ! __kmp_cpuinfo.initialized ) {
688 __kmp_query_cpuid( & __kmp_cpuinfo );
693 #if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB 708 __kmp_tls_gtid_min = 0;
710 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
714 if ( !__kmp_gtid_threadprivate_key ) {
715 __kmp_gtid_threadprivate_key = TlsAlloc();
716 if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
717 KMP_FATAL( TLSOutOfIndexes );
732 __kmp_str_buf_init( & path );
733 path_size = GetSystemDirectory( path.str, path.size );
734 KMP_DEBUG_ASSERT( path_size > 0 );
735 if ( path_size >= path.size ) {
739 __kmp_str_buf_reserve( & path, path_size );
740 path_size = GetSystemDirectory( path.str, path.size );
741 KMP_DEBUG_ASSERT( path_size > 0 );
743 if ( path_size > 0 && path_size < path.size ) {
748 path.used = path_size;
749 __kmp_str_buf_print( & path,
"\\%s",
"ntdll.dll" );
754 ntdll = GetModuleHandle( path.str );
757 KMP_DEBUG_ASSERT( ntdll != NULL );
758 if ( ntdll != NULL ) {
759 NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll,
"NtQuerySystemInformation" );
761 KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
763 #if KMP_GROUP_AFFINITY 768 if ( path_size > 0 && path_size < path.size ) {
773 path.used = path_size;
774 __kmp_str_buf_print( & path,
"\\%s",
"kernel32.dll" );
779 kernel32 = GetModuleHandle( path.str );
780 KA_TRACE( 10, (
"__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) );
786 if ( kernel32 != NULL ) {
787 __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32,
"GetActiveProcessorCount" );
788 __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32,
"GetActiveProcessorGroupCount" );
789 __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32,
"GetThreadGroupAffinity" );
790 __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32,
"SetThreadGroupAffinity" );
792 KA_TRACE( 10, (
"__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) );
793 KA_TRACE( 10, (
"__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) );
794 KA_TRACE( 10, (
"__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) );
795 KA_TRACE( 10, (
"__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) );
796 KA_TRACE( 10, (
"__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
sizeof(kmp_affin_mask_t) ) );
805 if ( ( __kmp_GetActiveProcessorCount != NULL )
806 && ( __kmp_GetActiveProcessorGroupCount != NULL )
807 && ( __kmp_GetThreadGroupAffinity != NULL )
808 && ( __kmp_SetThreadGroupAffinity != NULL )
809 && ( ( __kmp_num_proc_groups
810 = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
816 KA_TRACE( 10, (
"__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
820 for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
821 DWORD size = __kmp_GetActiveProcessorCount( i );
823 KA_TRACE( 10, (
"__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
827 KA_TRACE( 10, (
"__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
831 if ( __kmp_num_proc_groups <= 1 ) {
832 GetSystemInfo( & info );
833 __kmp_xproc = info.dwNumberOfProcessors;
836 GetSystemInfo( & info );
837 __kmp_xproc = info.dwNumberOfProcessors;
844 if ( __kmp_xproc <= 0 ) {
848 KA_TRACE( 5, (
"__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
850 __kmp_str_buf_free( & path );
853 __kmp_itt_initialize();
856 __kmp_init_runtime = TRUE;
860 __kmp_runtime_destroy(
void )
862 if ( ! __kmp_init_runtime ) {
872 KA_TRACE( 40, (
"__kmp_runtime_destroy\n" ));
874 if( __kmp_gtid_threadprivate_key ) {
875 TlsFree( __kmp_gtid_threadprivate_key );
876 __kmp_gtid_threadprivate_key = 0;
879 __kmp_affinity_uninitialize();
880 DeleteCriticalSection( & __kmp_win32_section );
883 NtQuerySystemInformation = NULL;
887 __kmp_GetActiveProcessorCount = NULL;
888 __kmp_GetActiveProcessorGroupCount = NULL;
889 __kmp_GetThreadGroupAffinity = NULL;
890 __kmp_SetThreadGroupAffinity = NULL;
891 #endif // KMP_ARCH_X86_64 893 __kmp_init_runtime = FALSE;
898 __kmp_terminate_thread(
int gtid )
900 kmp_info_t *th = __kmp_threads[ gtid ];
904 KA_TRACE( 10, (
"__kmp_terminate_thread: kill (%d)\n", gtid ) );
906 if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
909 __kmp_free_handle( th->th.th_info.ds.ds_thread );
916 __kmp_clear_system_time(
void )
920 status = QueryPerformanceCounter( & time );
921 __kmp_win32_time = (kmp_int64) time.QuadPart;
925 __kmp_initialize_system_tick(
void )
931 status = QueryPerformanceFrequency( & freq );
933 DWORD error = GetLastError();
936 KMP_MSG( FunctionError,
"QueryPerformanceFrequency()" ),
943 __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
951 __kmp_elapsed(
double *t )
955 status = QueryPerformanceCounter( & now );
956 *t = ((double) now.QuadPart) * __kmp_win32_tick;
962 __kmp_elapsed_tick(
double *t )
964 *t = __kmp_win32_tick;
968 __kmp_read_system_time(
double *delta )
974 status = QueryPerformanceCounter( & now );
976 *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
986 QueryPerformanceCounter(&now);
987 return 1e9 * __kmp_win32_tick * now.QuadPart;
994 __kmp_launch_worker(
void *arg )
996 volatile void *stack_data;
999 kmp_info_t *this_thr = (kmp_info_t *) arg;
1002 gtid = this_thr->th.th_info.ds.ds_gtid;
1003 __kmp_gtid_set_specific( gtid );
1004 #ifdef KMP_TDATA_GTID 1005 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 1006 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 1007 "reference: http://support.microsoft.com/kb/118816" 1012 __kmp_itt_thread_name( gtid );
1015 __kmp_affinity_set_init_mask( gtid, FALSE );
1017 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1022 __kmp_clear_x87_fpu_status_word();
1023 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
1024 __kmp_load_mxcsr( &__kmp_init_mxcsr );
1027 if ( __kmp_stkoffset > 0 && gtid > 0 ) {
1028 padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
1031 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1032 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1033 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1035 if ( TCR_4(__kmp_gtid_mode) < 2 ) {
1036 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
1037 KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
1038 __kmp_check_stack_overlap( this_thr );
1041 exit_val = __kmp_launch_thread( this_thr );
1042 KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
1043 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1052 __kmp_launch_monitor(
void *arg )
1055 kmp_thread_t monitor;
1058 kmp_info_t *this_thr = (kmp_info_t *) arg;
1060 KMP_DEBUG_ASSERT(__kmp_init_monitor);
1061 TCW_4( __kmp_init_monitor, 2 );
1063 this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1064 TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
1067 KA_TRACE( 10, (
"__kmp_launch_monitor: launched\n" ) );
1069 monitor = GetCurrentThread();
1072 status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
1074 DWORD error = GetLastError();
1077 KMP_MSG( CantSetThreadPriority ),
1084 __kmp_gtid_set_specific( KMP_GTID_MONITOR );
1085 #ifdef KMP_TDATA_GTID 1086 #error "This define causes problems with LoadLibrary() + declspec(thread) " \ 1087 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \ 1088 "reference: http://support.microsoft.com/kb/118816" 1093 __kmp_itt_thread_ignore();
1098 interval = ( 1000 / __kmp_monitor_wakeups );
1100 while (! TCR_4(__kmp_global.g.g_done)) {
1103 KA_TRACE( 15, (
"__kmp_launch_monitor: update\n" ) );
1105 wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
1107 if (wait_status == WAIT_TIMEOUT) {
1108 TCW_4( __kmp_global.g.g_time.dt.t_value,
1109 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
1115 KA_TRACE( 10, (
"__kmp_launch_monitor: finished\n" ) );
1117 status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
1119 DWORD error = GetLastError();
1122 KMP_MSG( CantSetThreadPriority ),
1128 if (__kmp_global.g.g_abort != 0) {
1134 KA_TRACE( 10, (
"__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
1139 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1140 __kmp_terminate_thread( gtid );
1146 KA_TRACE( 10, (
"__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
1148 if (__kmp_global.g.g_abort > 0) {
1149 raise( __kmp_global.g.g_abort );
1153 TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
1161 __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size )
1163 kmp_thread_t handle;
1166 KA_TRACE( 10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1168 th->th.th_info.ds.ds_gtid = gtid;
1170 if ( KMP_UBER_GTID(gtid) ) {
1179 rc = DuplicateHandle(
1180 GetCurrentProcess(),
1182 GetCurrentProcess(),
1183 &th->th.th_info.ds.ds_thread,
1186 DUPLICATE_SAME_ACCESS
1189 KA_TRACE( 10, (
" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC
"\n",
1191 th->th.th_info.ds.ds_thread ) );
1192 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1194 if ( TCR_4(__kmp_gtid_mode) < 2 ) {
1196 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1197 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1198 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1199 __kmp_check_stack_overlap( th );
1206 KA_TRACE( 10, (
"__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
1207 " bytes\n", stack_size ) );
1209 stack_size += gtid * __kmp_stkoffset;
1211 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1212 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1214 KA_TRACE( 10, (
"__kmp_create_worker: (before) stack_size = %" 1216 " bytes, &__kmp_launch_worker = %p, th = %p, " 1218 (SIZE_T) stack_size,
1219 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1220 (LPVOID) th, &idThread ) );
1222 handle = CreateThread( NULL, (SIZE_T) stack_size,
1223 (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
1224 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1226 KA_TRACE( 10, (
"__kmp_create_worker: (after) stack_size = %" 1228 " bytes, &__kmp_launch_worker = %p, th = %p, " 1229 "idThread = %u, handle = %" KMP_UINTPTR_SPEC
"\n",
1230 (SIZE_T) stack_size,
1231 (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
1232 (LPVOID) th, idThread, handle ) );
1234 if ( handle == 0 ) {
1235 DWORD error = GetLastError();
1236 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantCreateThread ), KMP_ERR( error ), __kmp_msg_null);
1238 th->th.th_info.ds.ds_thread = handle;
1244 KA_TRACE( 10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1248 __kmp_still_running(kmp_info_t *th) {
1249 return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
1254 __kmp_create_monitor( kmp_info_t *th )
1256 kmp_thread_t handle;
1258 int ideal, new_ideal;
1260 if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
1262 KA_TRACE( 10, (
"__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
1263 th->th.th_info.ds.ds_tid = 0;
1264 th->th.th_info.ds.ds_gtid = 0;
1265 TCW_4( __kmp_init_monitor, 2 );
1268 KA_TRACE( 10, (
"__kmp_create_monitor: try to create monitor\n" ) );
1272 __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
1273 if ( __kmp_monitor_ev == NULL ) {
1274 DWORD error = GetLastError();
1277 KMP_MSG( CantCreateEvent ),
1283 __kmp_itt_system_object_created( __kmp_monitor_ev,
"Event" );
1286 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1287 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1291 if ( __kmp_monitor_stksize == 0 ) {
1292 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1294 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1295 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1298 KA_TRACE( 10, (
"__kmp_create_monitor: requested stacksize = %d bytes\n",
1299 (
int) __kmp_monitor_stksize ) );
1301 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1303 handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
1304 (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
1305 (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
1307 DWORD error = GetLastError();
1310 KMP_MSG( CantCreateThread ),
1316 th->th.th_info.ds.ds_thread = handle;
1320 KA_TRACE( 10, (
"__kmp_create_monitor: monitor created %p\n",
1321 (
void *) th->th.th_info.ds.ds_thread ) );
1335 __kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
1338 rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
1340 DWORD error = GetLastError();
1343 KMP_MSG( FunctionError,
"GetExitCodeThread()" ),
1348 return ( *exit_val == STILL_ACTIVE );
1356 ExitThread( exit_status );
1363 __kmp_reap_common( kmp_info_t * th )
1369 KA_TRACE( 10, (
"__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
1389 register kmp_uint32 spins;
1391 KMP_FSYNC_SPIN_INIT( obj, (
void*) & th->th.th_info.ds.ds_alive );
1393 KMP_INIT_YIELD( spins );
1396 KMP_FSYNC_SPIN_PREPARE( obj );
1398 __kmp_is_thread_alive( th, &exit_val );
1399 KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
1400 KMP_YIELD_SPIN( spins );
1401 }
while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
1403 if ( exit_val == STILL_ACTIVE ) {
1404 KMP_FSYNC_CANCEL( obj );
1406 KMP_FSYNC_SPIN_ACQUIRED( obj );
1411 __kmp_free_handle( th->th.th_info.ds.ds_thread );
1418 if ( exit_val == STILL_ACTIVE ) {
1419 KA_TRACE( 1, (
"__kmp_reap_common: thread still active.\n" ) );
1420 }
else if ( (
void *) exit_val != (
void *) th) {
1421 KA_TRACE( 1, (
"__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
1426 "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
"\n",
1427 th->th.th_info.ds.ds_gtid,
1428 th->th.th_info.ds.ds_thread
1432 th->th.th_info.ds.ds_thread = 0;
1433 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1434 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1435 th->th.th_info.ds.ds_thread_id = 0;
1442 __kmp_reap_monitor( kmp_info_t *th )
1446 KA_TRACE( 10, (
"__kmp_reap_monitor: try to reap %p\n",
1447 (
void *) th->th.th_info.ds.ds_thread ) );
1452 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1453 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1454 KA_TRACE( 10, (
"__kmp_reap_monitor: monitor did not start, returning\n") );
1460 status = SetEvent( __kmp_monitor_ev );
1461 if ( status == FALSE ) {
1462 DWORD error = GetLastError();
1465 KMP_MSG( CantSetEvent ),
1470 KA_TRACE( 10, (
"__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1471 __kmp_reap_common( th );
1473 __kmp_free_handle( __kmp_monitor_ev );
1480 __kmp_reap_worker( kmp_info_t * th )
1482 KA_TRACE( 10, (
"__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
1483 __kmp_reap_common( th );
1489 #if KMP_HANDLE_SIGNALS 1493 __kmp_team_handler(
int signo )
1495 if ( __kmp_global.g.g_abort == 0 ) {
1497 if ( __kmp_debug_buf ) {
1498 __kmp_dump_debug_buffer();
1501 TCW_4( __kmp_global.g.g_abort, signo );
1503 TCW_4( __kmp_global.g.g_done, TRUE );
1511 sig_func_t __kmp_signal(
int signum, sig_func_t handler ) {
1512 sig_func_t old = signal( signum, handler );
1513 if ( old == SIG_ERR ) {
1515 __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError,
"signal" ), KMP_ERR( error ), __kmp_msg_null );
1521 __kmp_install_one_handler(
1528 KB_TRACE( 60, (
"__kmp_install_one_handler: called: sig=%d\n", sig ) );
1529 if ( parallel_init ) {
1530 old = __kmp_signal( sig, handler );
1532 if ( old == __kmp_sighldrs[ sig ] ) {
1533 __kmp_siginstalled[ sig ] = 1;
1536 old = __kmp_signal( sig, old );
1542 old = __kmp_signal( sig, SIG_DFL );
1543 __kmp_sighldrs[ sig ] = old;
1544 __kmp_signal( sig, old );
1550 __kmp_remove_one_handler(
int sig ) {
1551 if ( __kmp_siginstalled[ sig ] ) {
1554 KB_TRACE( 60, (
"__kmp_remove_one_handler: called: sig=%d\n", sig ) );
1555 old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
1556 if ( old != __kmp_team_handler ) {
1557 KB_TRACE( 10, (
"__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1558 old = __kmp_signal( sig, old );
1560 __kmp_sighldrs[ sig ] = NULL;
1561 __kmp_siginstalled[ sig ] = 0;
1568 __kmp_install_signals(
int parallel_init )
1570 KB_TRACE( 10, (
"__kmp_install_signals: called\n" ) );
1571 if ( ! __kmp_handle_signals ) {
1572 KB_TRACE( 10, (
"__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
1575 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1576 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1577 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1578 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1579 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1580 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1585 __kmp_remove_signals(
void )
1588 KB_TRACE( 10, (
"__kmp_remove_signals: called\n" ) );
1589 for ( sig = 1; sig < NSIG; ++ sig ) {
1590 __kmp_remove_one_handler( sig );
1595 #endif // KMP_HANDLE_SIGNALS 1599 __kmp_thread_sleep(
int millis )
1603 status = SleepEx( (DWORD) millis, FALSE );
1605 DWORD error = GetLastError();
1608 KMP_MSG( FunctionError,
"SleepEx()" ),
1617 __kmp_is_address_mapped(
void * addr )
1620 MEMORY_BASIC_INFORMATION lpBuffer;
1623 dwLength =
sizeof(MEMORY_BASIC_INFORMATION);
1625 status = VirtualQuery( addr, &lpBuffer, dwLength );
1627 return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
1628 (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
1632 __kmp_hardware_timestamp(
void)
1636 QueryPerformanceCounter((LARGE_INTEGER*) &r);
1642 __kmp_free_handle( kmp_thread_t tHandle )
1646 rc = CloseHandle( tHandle );
1648 DWORD error = GetLastError();
1651 KMP_MSG( CantCloseHandle ),
1659 __kmp_get_load_balance(
int max ) {
1661 static ULONG glb_buff_size = 100 * 1024;
1663 static int glb_running_threads = 0;
1664 static double glb_call_time = 0;
1666 int running_threads = 0;
1667 NTSTATUS status = 0;
1668 ULONG buff_size = 0;
1669 ULONG info_size = 0;
1670 void * buffer = NULL;
1671 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1674 double call_time = 0.0;
1676 __kmp_elapsed( & call_time );
1678 if ( glb_call_time &&
1679 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
1680 running_threads = glb_running_threads;
1683 glb_call_time = call_time;
1686 if ( NtQuerySystemInformation == NULL ) {
1687 running_threads = -1;
1698 buff_size = glb_buff_size;
1700 buff_size = 2 * buff_size;
1703 buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
1704 if ( buffer == NULL ) {
1705 running_threads = -1;
1708 status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
1711 }
while ( status == STATUS_INFO_LENGTH_MISMATCH );
1712 glb_buff_size = buff_size;
1714 #define CHECK( cond ) \ 1716 KMP_DEBUG_ASSERT( cond ); \ 1717 if ( ! ( cond ) ) { \ 1718 running_threads = -1; \ 1723 CHECK( buff_size >= info_size );
1724 spi = PSYSTEM_PROCESS_INFORMATION( buffer );
1726 ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
1727 CHECK( 0 <= offset && offset +
sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
1728 HANDLE pid = spi->ProcessId;
1729 ULONG num = spi->NumberOfThreads;
1731 size_t spi_size =
sizeof( SYSTEM_PROCESS_INFORMATION ) +
sizeof( SYSTEM_THREAD ) * ( num - 1 );
1732 CHECK( offset + spi_size < info_size );
1733 if ( spi->NextEntryOffset != 0 ) {
1734 CHECK( spi_size <= spi->NextEntryOffset );
1739 for (
int i = 0; i < num; ++ i ) {
1740 THREAD_STATE state = spi->Threads[ i ].State;
1743 if ( state == StateRunning ) {
1747 if ( running_threads >= max ) {
1753 if ( spi->NextEntryOffset == 0 ) {
1756 spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
1763 if ( buffer != NULL ) {
1764 KMP_INTERNAL_FREE( buffer );
1767 glb_running_threads = running_threads;
1769 return running_threads;