25 #include "tsan_annotations.h" 29 # include <sys/syscall.h> 47 __kmp_validate_locks(
void )
53 x = ~((kmp_uint32) 0) - 2;
56 for (i = 0; i < 8; ++i, ++x, ++y) {
57 kmp_uint32 z = (x - y);
61 KMP_ASSERT( offsetof( kmp_base_queuing_lock, tail_id ) % 8 == 0 );
79 __kmp_get_tas_lock_owner( kmp_tas_lock_t *lck )
81 return KMP_LOCK_STRIP(TCR_4( lck->lk.poll )) - 1;
85 __kmp_is_tas_lock_nestable( kmp_tas_lock_t *lck )
87 return lck->lk.depth_locked != -1;
90 __forceinline
static int 91 __kmp_acquire_tas_lock_timed_template( kmp_tas_lock_t *lck, kmp_int32 gtid )
95 #ifdef USE_LOCK_PROFILE 96 kmp_uint32 curr = KMP_LOCK_STRIP( TCR_4( lck->lk.poll ) );
97 if ( ( curr != 0 ) && ( curr != gtid + 1 ) )
98 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
102 if ( ( lck->lk.poll == KMP_LOCK_FREE(tas) )
103 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) {
104 KMP_FSYNC_ACQUIRED(lck);
105 return KMP_LOCK_ACQUIRED_FIRST;
109 KMP_FSYNC_PREPARE( lck );
110 KMP_INIT_YIELD( spins );
111 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
116 KMP_YIELD_SPIN( spins );
119 kmp_backoff_t backoff = __kmp_spin_backoff_params;
120 while ( ( lck->lk.poll != KMP_LOCK_FREE(tas) ) ||
121 ( ! KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) ) {
123 __kmp_spin_backoff(&backoff);
124 if ( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
129 KMP_YIELD_SPIN( spins );
132 KMP_FSYNC_ACQUIRED( lck );
133 return KMP_LOCK_ACQUIRED_FIRST;
137 __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
139 int retval = __kmp_acquire_tas_lock_timed_template( lck, gtid );
140 ANNOTATE_TAS_ACQUIRED(lck);
145 __kmp_acquire_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
147 char const *
const func =
"omp_set_lock";
148 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
149 && __kmp_is_tas_lock_nestable( lck ) ) {
150 KMP_FATAL( LockNestableUsedAsSimple, func );
152 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) == gtid ) ) {
153 KMP_FATAL( LockIsAlreadyOwned, func );
155 return __kmp_acquire_tas_lock( lck, gtid );
159 __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
161 if ( ( lck->lk.poll == KMP_LOCK_FREE(tas) )
162 && KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas) ) ) {
163 KMP_FSYNC_ACQUIRED( lck );
170 __kmp_test_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
172 char const *
const func =
"omp_test_lock";
173 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
174 && __kmp_is_tas_lock_nestable( lck ) ) {
175 KMP_FATAL( LockNestableUsedAsSimple, func );
177 return __kmp_test_tas_lock( lck, gtid );
181 __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
185 KMP_FSYNC_RELEASING(lck);
186 ANNOTATE_TAS_RELEASED(lck);
187 KMP_ST_REL32( &(lck->lk.poll), KMP_LOCK_FREE(tas) );
190 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
192 return KMP_LOCK_RELEASED;
196 __kmp_release_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
198 char const *
const func =
"omp_unset_lock";
200 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
201 && __kmp_is_tas_lock_nestable( lck ) ) {
202 KMP_FATAL( LockNestableUsedAsSimple, func );
204 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
205 KMP_FATAL( LockUnsettingFree, func );
207 if ( ( gtid >= 0 ) && ( __kmp_get_tas_lock_owner( lck ) >= 0 )
208 && ( __kmp_get_tas_lock_owner( lck ) != gtid ) ) {
209 KMP_FATAL( LockUnsettingSetByAnother, func );
211 return __kmp_release_tas_lock( lck, gtid );
215 __kmp_init_tas_lock( kmp_tas_lock_t * lck )
217 TCW_4( lck->lk.poll, KMP_LOCK_FREE(tas) );
221 __kmp_init_tas_lock_with_checks( kmp_tas_lock_t * lck )
223 __kmp_init_tas_lock( lck );
227 __kmp_destroy_tas_lock( kmp_tas_lock_t *lck )
233 __kmp_destroy_tas_lock_with_checks( kmp_tas_lock_t *lck )
235 char const *
const func =
"omp_destroy_lock";
236 if ( (
sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
237 && __kmp_is_tas_lock_nestable( lck ) ) {
238 KMP_FATAL( LockNestableUsedAsSimple, func );
240 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
241 KMP_FATAL( LockStillOwned, func );
243 __kmp_destroy_tas_lock( lck );
252 __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
254 KMP_DEBUG_ASSERT( gtid >= 0 );
256 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
257 lck->lk.depth_locked += 1;
258 return KMP_LOCK_ACQUIRED_NEXT;
261 __kmp_acquire_tas_lock_timed_template( lck, gtid );
262 ANNOTATE_TAS_ACQUIRED(lck);
263 lck->lk.depth_locked = 1;
264 return KMP_LOCK_ACQUIRED_FIRST;
269 __kmp_acquire_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
271 char const *
const func =
"omp_set_nest_lock";
272 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
273 KMP_FATAL( LockSimpleUsedAsNestable, func );
275 return __kmp_acquire_nested_tas_lock( lck, gtid );
279 __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
283 KMP_DEBUG_ASSERT( gtid >= 0 );
285 if ( __kmp_get_tas_lock_owner( lck ) == gtid ) {
286 retval = ++lck->lk.depth_locked;
288 else if ( !__kmp_test_tas_lock( lck, gtid ) ) {
293 retval = lck->lk.depth_locked = 1;
299 __kmp_test_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
301 char const *
const func =
"omp_test_nest_lock";
302 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
303 KMP_FATAL( LockSimpleUsedAsNestable, func );
305 return __kmp_test_nested_tas_lock( lck, gtid );
309 __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid )
311 KMP_DEBUG_ASSERT( gtid >= 0 );
314 if ( --(lck->lk.depth_locked) == 0 ) {
315 __kmp_release_tas_lock( lck, gtid );
316 return KMP_LOCK_RELEASED;
318 return KMP_LOCK_STILL_HELD;
322 __kmp_release_nested_tas_lock_with_checks( kmp_tas_lock_t *lck, kmp_int32 gtid )
324 char const *
const func =
"omp_unset_nest_lock";
326 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
327 KMP_FATAL( LockSimpleUsedAsNestable, func );
329 if ( __kmp_get_tas_lock_owner( lck ) == -1 ) {
330 KMP_FATAL( LockUnsettingFree, func );
332 if ( __kmp_get_tas_lock_owner( lck ) != gtid ) {
333 KMP_FATAL( LockUnsettingSetByAnother, func );
335 return __kmp_release_nested_tas_lock( lck, gtid );
339 __kmp_init_nested_tas_lock( kmp_tas_lock_t * lck )
341 __kmp_init_tas_lock( lck );
342 lck->lk.depth_locked = 0;
346 __kmp_init_nested_tas_lock_with_checks( kmp_tas_lock_t * lck )
348 __kmp_init_nested_tas_lock( lck );
352 __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck )
354 __kmp_destroy_tas_lock( lck );
355 lck->lk.depth_locked = 0;
359 __kmp_destroy_nested_tas_lock_with_checks( kmp_tas_lock_t *lck )
361 char const *
const func =
"omp_destroy_nest_lock";
362 if ( ! __kmp_is_tas_lock_nestable( lck ) ) {
363 KMP_FATAL( LockSimpleUsedAsNestable, func );
365 if ( __kmp_get_tas_lock_owner( lck ) != -1 ) {
366 KMP_FATAL( LockStillOwned, func );
368 __kmp_destroy_nested_tas_lock( lck );
383 __kmp_get_futex_lock_owner( kmp_futex_lock_t *lck )
385 return KMP_LOCK_STRIP(( TCR_4( lck->lk.poll ) >> 1 )) - 1;
389 __kmp_is_futex_lock_nestable( kmp_futex_lock_t *lck )
391 return lck->lk.depth_locked != -1;
394 __forceinline
static int 395 __kmp_acquire_futex_lock_timed_template( kmp_futex_lock_t *lck, kmp_int32 gtid )
397 kmp_int32 gtid_code = ( gtid + 1 ) << 1;
401 #ifdef USE_LOCK_PROFILE 402 kmp_uint32 curr = KMP_LOCK_STRIP( TCR_4( lck->lk.poll ) );
403 if ( ( curr != 0 ) && ( curr != gtid_code ) )
404 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
408 KMP_FSYNC_PREPARE( lck );
409 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
410 lck, lck->lk.poll, gtid ) );
414 while ( ( poll_val = KMP_COMPARE_AND_STORE_RET32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex),
415 KMP_LOCK_BUSY(gtid_code, futex) ) ) != KMP_LOCK_FREE(futex) ) {
417 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
418 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
419 lck, gtid, poll_val, cond ) );
434 if ( ! KMP_COMPARE_AND_STORE_REL32( & ( lck->lk.poll ), poll_val, poll_val | KMP_LOCK_BUSY(1, futex) ) ) {
435 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
436 lck, lck->lk.poll, gtid ) );
439 poll_val |= KMP_LOCK_BUSY(1, futex);
441 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n",
442 lck, lck->lk.poll, gtid ) );
445 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
446 lck, gtid, poll_val ) );
449 if ( ( rc = syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAIT,
450 poll_val, NULL, NULL, 0 ) ) != 0 ) {
451 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) failed (rc=%d errno=%d)\n",
452 lck, gtid, poll_val, rc, errno ) );
456 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
457 lck, gtid, poll_val ) );
467 KMP_FSYNC_ACQUIRED( lck );
468 KA_TRACE( 1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n",
469 lck, lck->lk.poll, gtid ) );
470 return KMP_LOCK_ACQUIRED_FIRST;
474 __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
476 int retval = __kmp_acquire_futex_lock_timed_template( lck, gtid );
477 ANNOTATE_FUTEX_ACQUIRED(lck);
482 __kmp_acquire_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
484 char const *
const func =
"omp_set_lock";
485 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
486 && __kmp_is_futex_lock_nestable( lck ) ) {
487 KMP_FATAL( LockNestableUsedAsSimple, func );
489 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) == gtid ) ) {
490 KMP_FATAL( LockIsAlreadyOwned, func );
492 return __kmp_acquire_futex_lock( lck, gtid );
496 __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
498 if ( KMP_COMPARE_AND_STORE_ACQ32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY((gtid+1) << 1, futex) ) ) {
499 KMP_FSYNC_ACQUIRED( lck );
506 __kmp_test_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
508 char const *
const func =
"omp_test_lock";
509 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
510 && __kmp_is_futex_lock_nestable( lck ) ) {
511 KMP_FATAL( LockNestableUsedAsSimple, func );
513 return __kmp_test_futex_lock( lck, gtid );
517 __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
521 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
522 lck, lck->lk.poll, gtid ) );
524 KMP_FSYNC_RELEASING(lck);
525 ANNOTATE_FUTEX_RELEASED(lck);
527 kmp_int32 poll_val = KMP_XCHG_FIXED32( & ( lck->lk.poll ), KMP_LOCK_FREE(futex) );
529 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
530 lck, gtid, poll_val ) );
532 if ( KMP_LOCK_STRIP(poll_val) & 1 ) {
533 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
535 syscall( __NR_futex, & ( lck->lk.poll ), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0 );
540 KA_TRACE( 1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n",
541 lck, lck->lk.poll, gtid ) );
543 KMP_YIELD( TCR_4( __kmp_nth ) > ( __kmp_avail_proc ? __kmp_avail_proc :
545 return KMP_LOCK_RELEASED;
549 __kmp_release_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
551 char const *
const func =
"omp_unset_lock";
553 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
554 && __kmp_is_futex_lock_nestable( lck ) ) {
555 KMP_FATAL( LockNestableUsedAsSimple, func );
557 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
558 KMP_FATAL( LockUnsettingFree, func );
560 if ( ( gtid >= 0 ) && ( __kmp_get_futex_lock_owner( lck ) >= 0 )
561 && ( __kmp_get_futex_lock_owner( lck ) != gtid ) ) {
562 KMP_FATAL( LockUnsettingSetByAnother, func );
564 return __kmp_release_futex_lock( lck, gtid );
568 __kmp_init_futex_lock( kmp_futex_lock_t * lck )
570 TCW_4( lck->lk.poll, KMP_LOCK_FREE(futex) );
574 __kmp_init_futex_lock_with_checks( kmp_futex_lock_t * lck )
576 __kmp_init_futex_lock( lck );
580 __kmp_destroy_futex_lock( kmp_futex_lock_t *lck )
586 __kmp_destroy_futex_lock_with_checks( kmp_futex_lock_t *lck )
588 char const *
const func =
"omp_destroy_lock";
589 if ( (
sizeof ( kmp_futex_lock_t ) <= OMP_LOCK_T_SIZE )
590 && __kmp_is_futex_lock_nestable( lck ) ) {
591 KMP_FATAL( LockNestableUsedAsSimple, func );
593 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
594 KMP_FATAL( LockStillOwned, func );
596 __kmp_destroy_futex_lock( lck );
605 __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
607 KMP_DEBUG_ASSERT( gtid >= 0 );
609 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
610 lck->lk.depth_locked += 1;
611 return KMP_LOCK_ACQUIRED_NEXT;
614 __kmp_acquire_futex_lock_timed_template( lck, gtid );
615 ANNOTATE_FUTEX_ACQUIRED(lck);
616 lck->lk.depth_locked = 1;
617 return KMP_LOCK_ACQUIRED_FIRST;
622 __kmp_acquire_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
624 char const *
const func =
"omp_set_nest_lock";
625 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
626 KMP_FATAL( LockSimpleUsedAsNestable, func );
628 return __kmp_acquire_nested_futex_lock( lck, gtid );
632 __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
636 KMP_DEBUG_ASSERT( gtid >= 0 );
638 if ( __kmp_get_futex_lock_owner( lck ) == gtid ) {
639 retval = ++lck->lk.depth_locked;
641 else if ( !__kmp_test_futex_lock( lck, gtid ) ) {
646 retval = lck->lk.depth_locked = 1;
652 __kmp_test_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
654 char const *
const func =
"omp_test_nest_lock";
655 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
656 KMP_FATAL( LockSimpleUsedAsNestable, func );
658 return __kmp_test_nested_futex_lock( lck, gtid );
662 __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid )
664 KMP_DEBUG_ASSERT( gtid >= 0 );
667 if ( --(lck->lk.depth_locked) == 0 ) {
668 __kmp_release_futex_lock( lck, gtid );
669 return KMP_LOCK_RELEASED;
671 return KMP_LOCK_STILL_HELD;
675 __kmp_release_nested_futex_lock_with_checks( kmp_futex_lock_t *lck, kmp_int32 gtid )
677 char const *
const func =
"omp_unset_nest_lock";
679 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
680 KMP_FATAL( LockSimpleUsedAsNestable, func );
682 if ( __kmp_get_futex_lock_owner( lck ) == -1 ) {
683 KMP_FATAL( LockUnsettingFree, func );
685 if ( __kmp_get_futex_lock_owner( lck ) != gtid ) {
686 KMP_FATAL( LockUnsettingSetByAnother, func );
688 return __kmp_release_nested_futex_lock( lck, gtid );
692 __kmp_init_nested_futex_lock( kmp_futex_lock_t * lck )
694 __kmp_init_futex_lock( lck );
695 lck->lk.depth_locked = 0;
699 __kmp_init_nested_futex_lock_with_checks( kmp_futex_lock_t * lck )
701 __kmp_init_nested_futex_lock( lck );
705 __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck )
707 __kmp_destroy_futex_lock( lck );
708 lck->lk.depth_locked = 0;
712 __kmp_destroy_nested_futex_lock_with_checks( kmp_futex_lock_t *lck )
714 char const *
const func =
"omp_destroy_nest_lock";
715 if ( ! __kmp_is_futex_lock_nestable( lck ) ) {
716 KMP_FATAL( LockSimpleUsedAsNestable, func );
718 if ( __kmp_get_futex_lock_owner( lck ) != -1 ) {
719 KMP_FATAL( LockStillOwned, func );
721 __kmp_destroy_nested_futex_lock( lck );
724 #endif // KMP_USE_FUTEX 731 __kmp_get_ticket_lock_owner( kmp_ticket_lock_t *lck )
733 return std::atomic_load_explicit( &lck->lk.owner_id, std::memory_order_relaxed ) - 1;
737 __kmp_is_ticket_lock_nestable( kmp_ticket_lock_t *lck )
739 return std::atomic_load_explicit( &lck->lk.depth_locked, std::memory_order_relaxed ) != -1;
743 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket )
745 return std::atomic_load_explicit( (std::atomic<unsigned> *)now_serving, std::memory_order_acquire ) == my_ticket;
748 __forceinline
static int 749 __kmp_acquire_ticket_lock_timed_template( kmp_ticket_lock_t *lck, kmp_int32 gtid )
751 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit( &lck->lk.next_ticket, 1U, std::memory_order_relaxed );
753 #ifdef USE_LOCK_PROFILE 754 if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed ) != my_ticket )
755 __kmp_printf(
"LOCK CONTENTION: %p\n", lck );
759 if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_acquire ) == my_ticket ) {
760 return KMP_LOCK_ACQUIRED_FIRST;
762 KMP_WAIT_YIELD_PTR( &lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck );
763 return KMP_LOCK_ACQUIRED_FIRST;
767 __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
769 int retval = __kmp_acquire_ticket_lock_timed_template( lck, gtid );
770 ANNOTATE_TICKET_ACQUIRED(lck);
775 __kmp_acquire_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
777 char const *
const func =
"omp_set_lock";
779 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
780 KMP_FATAL( LockIsUninitialized, func );
782 if ( lck->lk.self != lck ) {
783 KMP_FATAL( LockIsUninitialized, func );
785 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
786 KMP_FATAL( LockNestableUsedAsSimple, func );
788 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) == gtid ) ) {
789 KMP_FATAL( LockIsAlreadyOwned, func );
792 __kmp_acquire_ticket_lock( lck, gtid );
794 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
795 return KMP_LOCK_ACQUIRED_FIRST;
799 __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
801 kmp_uint32 my_ticket = std::atomic_load_explicit( &lck->lk.next_ticket, std::memory_order_relaxed );
803 if ( std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed ) == my_ticket ) {
804 kmp_uint32 next_ticket = my_ticket + 1;
805 if ( std::atomic_compare_exchange_strong_explicit( &lck->lk.next_ticket,
806 &my_ticket, next_ticket, std::memory_order_acquire, std::memory_order_acquire )) {
814 __kmp_test_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
816 char const *
const func =
"omp_test_lock";
818 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
819 KMP_FATAL( LockIsUninitialized, func );
821 if ( lck->lk.self != lck ) {
822 KMP_FATAL( LockIsUninitialized, func );
824 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
825 KMP_FATAL( LockNestableUsedAsSimple, func );
828 int retval = __kmp_test_ticket_lock( lck, gtid );
831 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
837 __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
839 kmp_uint32 distance = std::atomic_load_explicit( &lck->lk.next_ticket, std::memory_order_relaxed ) - std::atomic_load_explicit( &lck->lk.now_serving, std::memory_order_relaxed );
841 ANNOTATE_TICKET_RELEASED(lck);
842 std::atomic_fetch_add_explicit( &lck->lk.now_serving, 1U, std::memory_order_release );
845 > (kmp_uint32) (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) );
846 return KMP_LOCK_RELEASED;
850 __kmp_release_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
852 char const *
const func =
"omp_unset_lock";
854 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
855 KMP_FATAL( LockIsUninitialized, func );
857 if ( lck->lk.self != lck ) {
858 KMP_FATAL( LockIsUninitialized, func );
860 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
861 KMP_FATAL( LockNestableUsedAsSimple, func );
863 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
864 KMP_FATAL( LockUnsettingFree, func );
866 if ( ( gtid >= 0 ) && ( __kmp_get_ticket_lock_owner( lck ) >= 0 )
867 && ( __kmp_get_ticket_lock_owner( lck ) != gtid ) ) {
868 KMP_FATAL( LockUnsettingSetByAnother, func );
870 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
871 return __kmp_release_ticket_lock( lck, gtid );
875 __kmp_init_ticket_lock( kmp_ticket_lock_t * lck )
877 lck->lk.location = NULL;
879 std::atomic_store_explicit( &lck->lk.next_ticket, 0U, std::memory_order_relaxed );
880 std::atomic_store_explicit( &lck->lk.now_serving, 0U, std::memory_order_relaxed );
881 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
882 std::atomic_store_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed );
883 std::atomic_store_explicit( &lck->lk.initialized,
true, std::memory_order_release );
887 __kmp_init_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
889 __kmp_init_ticket_lock( lck );
893 __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck )
895 std::atomic_store_explicit( &lck->lk.initialized,
false, std::memory_order_release );
897 lck->lk.location = NULL;
898 std::atomic_store_explicit( &lck->lk.next_ticket, 0U, std::memory_order_relaxed );
899 std::atomic_store_explicit( &lck->lk.now_serving, 0U, std::memory_order_relaxed );
900 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
901 std::atomic_store_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed );
905 __kmp_destroy_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
907 char const *
const func =
"omp_destroy_lock";
909 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
910 KMP_FATAL( LockIsUninitialized, func );
912 if ( lck->lk.self != lck ) {
913 KMP_FATAL( LockIsUninitialized, func );
915 if ( __kmp_is_ticket_lock_nestable( lck ) ) {
916 KMP_FATAL( LockNestableUsedAsSimple, func );
918 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
919 KMP_FATAL( LockStillOwned, func );
921 __kmp_destroy_ticket_lock( lck );
930 __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
932 KMP_DEBUG_ASSERT( gtid >= 0 );
934 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
935 std::atomic_fetch_add_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
936 return KMP_LOCK_ACQUIRED_NEXT;
939 __kmp_acquire_ticket_lock_timed_template( lck, gtid );
940 ANNOTATE_TICKET_ACQUIRED(lck);
941 std::atomic_store_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
942 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
943 return KMP_LOCK_ACQUIRED_FIRST;
948 __kmp_acquire_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
950 char const *
const func =
"omp_set_nest_lock";
952 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
953 KMP_FATAL( LockIsUninitialized, func );
955 if ( lck->lk.self != lck ) {
956 KMP_FATAL( LockIsUninitialized, func );
958 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
959 KMP_FATAL( LockSimpleUsedAsNestable, func );
961 return __kmp_acquire_nested_ticket_lock( lck, gtid );
965 __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
969 KMP_DEBUG_ASSERT( gtid >= 0 );
971 if ( __kmp_get_ticket_lock_owner( lck ) == gtid ) {
972 retval = std::atomic_fetch_add_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed ) + 1;
974 else if ( !__kmp_test_ticket_lock( lck, gtid ) ) {
978 std::atomic_store_explicit( &lck->lk.depth_locked, 1, std::memory_order_relaxed );
979 std::atomic_store_explicit( &lck->lk.owner_id, gtid + 1, std::memory_order_relaxed );
986 __kmp_test_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck,
989 char const *
const func =
"omp_test_nest_lock";
991 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
992 KMP_FATAL( LockIsUninitialized, func );
994 if ( lck->lk.self != lck ) {
995 KMP_FATAL( LockIsUninitialized, func );
997 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
998 KMP_FATAL( LockSimpleUsedAsNestable, func );
1000 return __kmp_test_nested_ticket_lock( lck, gtid );
1004 __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1006 KMP_DEBUG_ASSERT( gtid >= 0 );
1008 if ( ( std::atomic_fetch_add_explicit( &lck->lk.depth_locked, -1, std::memory_order_relaxed ) - 1 ) == 0 ) {
1009 std::atomic_store_explicit( &lck->lk.owner_id, 0, std::memory_order_relaxed );
1010 __kmp_release_ticket_lock( lck, gtid );
1011 return KMP_LOCK_RELEASED;
1013 return KMP_LOCK_STILL_HELD;
1017 __kmp_release_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck, kmp_int32 gtid )
1019 char const *
const func =
"omp_unset_nest_lock";
1021 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
1022 KMP_FATAL( LockIsUninitialized, func );
1024 if ( lck->lk.self != lck ) {
1025 KMP_FATAL( LockIsUninitialized, func );
1027 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1028 KMP_FATAL( LockSimpleUsedAsNestable, func );
1030 if ( __kmp_get_ticket_lock_owner( lck ) == -1 ) {
1031 KMP_FATAL( LockUnsettingFree, func );
1033 if ( __kmp_get_ticket_lock_owner( lck ) != gtid ) {
1034 KMP_FATAL( LockUnsettingSetByAnother, func );
1036 return __kmp_release_nested_ticket_lock( lck, gtid );
1040 __kmp_init_nested_ticket_lock( kmp_ticket_lock_t * lck )
1042 __kmp_init_ticket_lock( lck );
1043 std::atomic_store_explicit( &lck->lk.depth_locked, 0, std::memory_order_relaxed );
1047 __kmp_init_nested_ticket_lock_with_checks( kmp_ticket_lock_t * lck )
1049 __kmp_init_nested_ticket_lock( lck );
1053 __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck )
1055 __kmp_destroy_ticket_lock( lck );
1056 std::atomic_store_explicit( &lck->lk.depth_locked, 0, std::memory_order_relaxed );
1060 __kmp_destroy_nested_ticket_lock_with_checks( kmp_ticket_lock_t *lck )
1062 char const *
const func =
"omp_destroy_nest_lock";
1064 if ( ! std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) ) {
1065 KMP_FATAL( LockIsUninitialized, func );
1067 if ( lck->lk.self != lck ) {
1068 KMP_FATAL( LockIsUninitialized, func );
1070 if ( ! __kmp_is_ticket_lock_nestable( lck ) ) {
1071 KMP_FATAL( LockSimpleUsedAsNestable, func );
1073 if ( __kmp_get_ticket_lock_owner( lck ) != -1 ) {
1074 KMP_FATAL( LockStillOwned, func );
1076 __kmp_destroy_nested_ticket_lock( lck );
1085 __kmp_is_ticket_lock_initialized( kmp_ticket_lock_t *lck )
1087 return std::atomic_load_explicit( &lck->lk.initialized, std::memory_order_relaxed ) && ( lck->lk.self == lck);
1091 __kmp_get_ticket_lock_location( kmp_ticket_lock_t *lck )
1093 return lck->lk.location;
1097 __kmp_set_ticket_lock_location( kmp_ticket_lock_t *lck,
const ident_t *loc )
1099 lck->lk.location = loc;
1102 static kmp_lock_flags_t
1103 __kmp_get_ticket_lock_flags( kmp_ticket_lock_t *lck )
1105 return lck->lk.flags;
1109 __kmp_set_ticket_lock_flags( kmp_ticket_lock_t *lck, kmp_lock_flags_t flags )
1111 lck->lk.flags = flags;
1170 #ifdef DEBUG_QUEUING_LOCKS 1173 #define TRACE_BUF_ELE 1024 1174 static char traces[TRACE_BUF_ELE][128] = { 0 }
1176 #define TRACE_LOCK(X,Y) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y ); 1177 #define TRACE_LOCK_T(X,Y,Z) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X,Y,Z ); 1178 #define TRACE_LOCK_HT(X,Y,Z,Q) KMP_SNPRINTF( traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, Z, Q ); 1181 __kmp_dump_queuing_lock( kmp_info_t *this_thr, kmp_int32 gtid,
1182 kmp_queuing_lock_t *lck, kmp_int32 head_id, kmp_int32 tail_id )
1186 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n" );
1188 i = tc % TRACE_BUF_ELE;
1189 __kmp_printf_no_lock(
"%s\n", traces[i] );
1190 i = (i+1) % TRACE_BUF_ELE;
1191 while ( i != (tc % TRACE_BUF_ELE) ) {
1192 __kmp_printf_no_lock(
"%s", traces[i] );
1193 i = (i+1) % TRACE_BUF_ELE;
1195 __kmp_printf_no_lock(
"\n" );
1197 __kmp_printf_no_lock(
1198 "\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, next_wait:%d, head_id:%d, tail_id:%d\n",
1199 gtid+1, this_thr->th.th_spin_here, this_thr->th.th_next_waiting,
1202 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id );
1204 if ( lck->lk.head_id >= 1 ) {
1205 t = __kmp_threads[lck->lk.head_id-1]->th.th_next_waiting;
1207 __kmp_printf_no_lock(
"-> %d ", t );
1208 t = __kmp_threads[t-1]->th.th_next_waiting;
1211 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id );
1212 __kmp_printf_no_lock(
"\n\n" );
1218 __kmp_get_queuing_lock_owner( kmp_queuing_lock_t *lck )
1220 return TCR_4( lck->lk.owner_id ) - 1;
1224 __kmp_is_queuing_lock_nestable( kmp_queuing_lock_t *lck )
1226 return lck->lk.depth_locked != -1;
1230 template <
bool takeTime>
1233 __forceinline
static int 1234 __kmp_acquire_queuing_lock_timed_template( kmp_queuing_lock_t *lck,
1237 register kmp_info_t *this_thr = __kmp_thread_from_gtid( gtid );
1238 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1239 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1240 volatile kmp_uint32 *spin_here_p;
1241 kmp_int32 need_mf = 1;
1244 ompt_state_t prev_state = ompt_state_undefined;
1247 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1249 KMP_FSYNC_PREPARE( lck );
1250 KMP_DEBUG_ASSERT( this_thr != NULL );
1251 spin_here_p = & this_thr->th.th_spin_here;
1253 #ifdef DEBUG_QUEUING_LOCKS 1254 TRACE_LOCK( gtid+1,
"acq ent" );
1256 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1257 if ( this_thr->th.th_next_waiting != 0 )
1258 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1260 KMP_DEBUG_ASSERT( !*spin_here_p );
1261 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1271 *spin_here_p = TRUE;
1284 #ifdef DEBUG_QUEUING_LOCKS 1286 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1295 enqueued = KMP_COMPARE_AND_STORE_ACQ64( (
volatile kmp_int64 *) tail_id_p,
1296 KMP_PACK_64( -1, 0 ),
1297 KMP_PACK_64( gtid+1, gtid+1 ) );
1298 #ifdef DEBUG_QUEUING_LOCKS 1299 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (-1,0)->(tid,tid)" );
1307 KMP_DEBUG_ASSERT( tail != gtid + 1 );
1309 #ifdef DEBUG_QUEUING_LOCKS 1310 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1319 enqueued = KMP_COMPARE_AND_STORE_ACQ32( tail_id_p, tail, gtid+1 );
1321 #ifdef DEBUG_QUEUING_LOCKS 1322 if ( enqueued ) TRACE_LOCK( gtid+1,
"acq enq: (h,t)->(h,tid)" );
1330 kmp_int32 grabbed_lock;
1332 #ifdef DEBUG_QUEUING_LOCKS 1334 TRACE_LOCK_HT( gtid+1,
"acq read: ", head, tail );
1339 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 );
1341 if ( grabbed_lock ) {
1343 *spin_here_p = FALSE;
1345 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1347 #ifdef DEBUG_QUEUING_LOCKS 1348 TRACE_LOCK_HT( gtid+1,
"acq exit: ", head, 0 );
1352 if (ompt_enabled && prev_state != ompt_state_undefined) {
1354 this_thr->th.ompt_thread_info.state = prev_state;
1355 this_thr->th.ompt_thread_info.wait_id = 0;
1359 KMP_FSYNC_ACQUIRED( lck );
1360 return KMP_LOCK_ACQUIRED_FIRST;
1368 if (ompt_enabled && prev_state == ompt_state_undefined) {
1370 prev_state = this_thr->th.ompt_thread_info.state;
1371 this_thr->th.ompt_thread_info.wait_id = (uint64_t) lck;
1372 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1378 kmp_info_t *tail_thr = __kmp_thread_from_gtid( tail - 1 );
1379 KMP_ASSERT( tail_thr != NULL );
1380 tail_thr->th.th_next_waiting = gtid+1;
1383 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n", lck, gtid ));
1390 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1392 #ifdef DEBUG_QUEUING_LOCKS 1393 TRACE_LOCK( gtid+1,
"acq spin" );
1395 if ( this_thr->th.th_next_waiting != 0 )
1396 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1398 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1399 KA_TRACE( 1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after waiting on queue\n",
1402 #ifdef DEBUG_QUEUING_LOCKS 1403 TRACE_LOCK( gtid+1,
"acq exit 2" );
1408 this_thr->th.ompt_thread_info.state = prev_state;
1409 this_thr->th.ompt_thread_info.wait_id = 0;
1413 return KMP_LOCK_ACQUIRED_FIRST;
1419 KMP_YIELD( TCR_4( __kmp_nth ) > (__kmp_avail_proc ? __kmp_avail_proc :
1421 #ifdef DEBUG_QUEUING_LOCKS 1422 TRACE_LOCK( gtid+1,
"acq retry" );
1426 KMP_ASSERT2( 0,
"should not get here" );
1427 return KMP_LOCK_ACQUIRED_FIRST;
1431 __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1433 KMP_DEBUG_ASSERT( gtid >= 0 );
1435 int retval = __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1436 ANNOTATE_QUEUING_ACQUIRED(lck);
1441 __kmp_acquire_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1444 char const *
const func =
"omp_set_lock";
1445 if ( lck->lk.initialized != lck ) {
1446 KMP_FATAL( LockIsUninitialized, func );
1448 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1449 KMP_FATAL( LockNestableUsedAsSimple, func );
1451 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1452 KMP_FATAL( LockIsAlreadyOwned, func );
1455 __kmp_acquire_queuing_lock( lck, gtid );
1457 lck->lk.owner_id = gtid + 1;
1458 return KMP_LOCK_ACQUIRED_FIRST;
1462 __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1464 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1467 kmp_info_t *this_thr;
1470 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid ));
1471 KMP_DEBUG_ASSERT( gtid >= 0 );
1473 this_thr = __kmp_thread_from_gtid( gtid );
1474 KMP_DEBUG_ASSERT( this_thr != NULL );
1475 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1484 if ( KMP_COMPARE_AND_STORE_ACQ32( head_id_p, 0, -1 ) ) {
1485 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid ));
1486 KMP_FSYNC_ACQUIRED(lck);
1487 ANNOTATE_QUEUING_ACQUIRED(lck);
1492 KA_TRACE( 1000, (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid ));
1497 __kmp_test_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1499 char const *
const func =
"omp_test_lock";
1500 if ( lck->lk.initialized != lck ) {
1501 KMP_FATAL( LockIsUninitialized, func );
1503 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1504 KMP_FATAL( LockNestableUsedAsSimple, func );
1507 int retval = __kmp_test_queuing_lock( lck, gtid );
1510 lck->lk.owner_id = gtid + 1;
1516 __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1518 register kmp_info_t *this_thr;
1519 volatile kmp_int32 *head_id_p = & lck->lk.head_id;
1520 volatile kmp_int32 *tail_id_p = & lck->lk.tail_id;
1522 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid ));
1523 KMP_DEBUG_ASSERT( gtid >= 0 );
1524 this_thr = __kmp_thread_from_gtid( gtid );
1525 KMP_DEBUG_ASSERT( this_thr != NULL );
1526 #ifdef DEBUG_QUEUING_LOCKS 1527 TRACE_LOCK( gtid+1,
"rel ent" );
1529 if ( this_thr->th.th_spin_here )
1530 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1531 if ( this_thr->th.th_next_waiting != 0 )
1532 __kmp_dump_queuing_lock( this_thr, gtid, lck, *head_id_p, *tail_id_p );
1534 KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
1535 KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
1537 KMP_FSYNC_RELEASING(lck);
1538 ANNOTATE_QUEUING_RELEASED(lck);
1547 #ifdef DEBUG_QUEUING_LOCKS 1549 TRACE_LOCK_HT( gtid+1,
"rel read: ", head, tail );
1550 if ( head == 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1552 KMP_DEBUG_ASSERT( head != 0 );
1557 if ( KMP_COMPARE_AND_STORE_REL32( head_id_p, -1, 0 ) ) {
1558 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1560 #ifdef DEBUG_QUEUING_LOCKS 1561 TRACE_LOCK_HT( gtid+1,
"rel exit: ", 0, 0 );
1568 return KMP_LOCK_RELEASED;
1576 if ( head == tail ) {
1578 #ifdef DEBUG_QUEUING_LOCKS 1579 if ( head <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1581 KMP_DEBUG_ASSERT( head > 0 );
1584 dequeued = KMP_COMPARE_AND_STORE_REL64( (kmp_int64 *) tail_id_p,
1585 KMP_PACK_64( head, head ), KMP_PACK_64( -1, 0 ) );
1586 #ifdef DEBUG_QUEUING_LOCKS 1587 TRACE_LOCK( gtid+1,
"rel deq: (h,h)->(-1,0)" );
1592 volatile kmp_int32 *waiting_id_p;
1593 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1594 KMP_DEBUG_ASSERT( head_thr != NULL );
1595 waiting_id_p = & head_thr->th.th_next_waiting;
1598 #ifdef DEBUG_QUEUING_LOCKS 1599 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1601 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1607 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32*)waiting_id_p, 0, KMP_NEQ, NULL);
1608 #ifdef DEBUG_QUEUING_LOCKS 1609 TRACE_LOCK( gtid+1,
"rel deq: (h,t)->(h',t)" );
1616 kmp_info_t *head_thr = __kmp_thread_from_gtid( head - 1 );
1617 KMP_DEBUG_ASSERT( head_thr != NULL );
1620 #ifdef DEBUG_QUEUING_LOCKS 1621 if ( head <= 0 || tail <= 0 ) __kmp_dump_queuing_lock( this_thr, gtid, lck, head, tail );
1623 KMP_DEBUG_ASSERT( head > 0 && tail > 0 );
1628 head_thr->th.th_next_waiting = 0;
1629 #ifdef DEBUG_QUEUING_LOCKS 1630 TRACE_LOCK_T( gtid+1,
"rel nw=0 for t=", head );
1635 head_thr->th.th_spin_here = FALSE;
1637 KA_TRACE( 1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after dequeuing\n",
1639 #ifdef DEBUG_QUEUING_LOCKS 1640 TRACE_LOCK( gtid+1,
"rel exit 2" );
1642 return KMP_LOCK_RELEASED;
1646 #ifdef DEBUG_QUEUING_LOCKS 1647 TRACE_LOCK( gtid+1,
"rel retry" );
1651 KMP_ASSERT2( 0,
"should not get here" );
1652 return KMP_LOCK_RELEASED;
1656 __kmp_release_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1659 char const *
const func =
"omp_unset_lock";
1661 if ( lck->lk.initialized != lck ) {
1662 KMP_FATAL( LockIsUninitialized, func );
1664 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1665 KMP_FATAL( LockNestableUsedAsSimple, func );
1667 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1668 KMP_FATAL( LockUnsettingFree, func );
1670 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1671 KMP_FATAL( LockUnsettingSetByAnother, func );
1673 lck->lk.owner_id = 0;
1674 return __kmp_release_queuing_lock( lck, gtid );
1678 __kmp_init_queuing_lock( kmp_queuing_lock_t *lck )
1680 lck->lk.location = NULL;
1681 lck->lk.head_id = 0;
1682 lck->lk.tail_id = 0;
1683 lck->lk.next_ticket = 0;
1684 lck->lk.now_serving = 0;
1685 lck->lk.owner_id = 0;
1686 lck->lk.depth_locked = -1;
1687 lck->lk.initialized = lck;
1689 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1693 __kmp_init_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1695 __kmp_init_queuing_lock( lck );
1699 __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck )
1701 lck->lk.initialized = NULL;
1702 lck->lk.location = NULL;
1703 lck->lk.head_id = 0;
1704 lck->lk.tail_id = 0;
1705 lck->lk.next_ticket = 0;
1706 lck->lk.now_serving = 0;
1707 lck->lk.owner_id = 0;
1708 lck->lk.depth_locked = -1;
1712 __kmp_destroy_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1714 char const *
const func =
"omp_destroy_lock";
1715 if ( lck->lk.initialized != lck ) {
1716 KMP_FATAL( LockIsUninitialized, func );
1718 if ( __kmp_is_queuing_lock_nestable( lck ) ) {
1719 KMP_FATAL( LockNestableUsedAsSimple, func );
1721 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1722 KMP_FATAL( LockStillOwned, func );
1724 __kmp_destroy_queuing_lock( lck );
1733 __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1735 KMP_DEBUG_ASSERT( gtid >= 0 );
1737 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1738 lck->lk.depth_locked += 1;
1739 return KMP_LOCK_ACQUIRED_NEXT;
1742 __kmp_acquire_queuing_lock_timed_template<false>( lck, gtid );
1743 ANNOTATE_QUEUING_ACQUIRED(lck);
1745 lck->lk.depth_locked = 1;
1747 lck->lk.owner_id = gtid + 1;
1748 return KMP_LOCK_ACQUIRED_FIRST;
1753 __kmp_acquire_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1755 char const *
const func =
"omp_set_nest_lock";
1756 if ( lck->lk.initialized != lck ) {
1757 KMP_FATAL( LockIsUninitialized, func );
1759 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1760 KMP_FATAL( LockSimpleUsedAsNestable, func );
1762 return __kmp_acquire_nested_queuing_lock( lck, gtid );
1766 __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1770 KMP_DEBUG_ASSERT( gtid >= 0 );
1772 if ( __kmp_get_queuing_lock_owner( lck ) == gtid ) {
1773 retval = ++lck->lk.depth_locked;
1775 else if ( !__kmp_test_queuing_lock( lck, gtid ) ) {
1780 retval = lck->lk.depth_locked = 1;
1782 lck->lk.owner_id = gtid + 1;
1788 __kmp_test_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck,
1791 char const *
const func =
"omp_test_nest_lock";
1792 if ( lck->lk.initialized != lck ) {
1793 KMP_FATAL( LockIsUninitialized, func );
1795 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1796 KMP_FATAL( LockSimpleUsedAsNestable, func );
1798 return __kmp_test_nested_queuing_lock( lck, gtid );
1802 __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1804 KMP_DEBUG_ASSERT( gtid >= 0 );
1807 if ( --(lck->lk.depth_locked) == 0 ) {
1809 lck->lk.owner_id = 0;
1810 __kmp_release_queuing_lock( lck, gtid );
1811 return KMP_LOCK_RELEASED;
1813 return KMP_LOCK_STILL_HELD;
1817 __kmp_release_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck, kmp_int32 gtid )
1819 char const *
const func =
"omp_unset_nest_lock";
1821 if ( lck->lk.initialized != lck ) {
1822 KMP_FATAL( LockIsUninitialized, func );
1824 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1825 KMP_FATAL( LockSimpleUsedAsNestable, func );
1827 if ( __kmp_get_queuing_lock_owner( lck ) == -1 ) {
1828 KMP_FATAL( LockUnsettingFree, func );
1830 if ( __kmp_get_queuing_lock_owner( lck ) != gtid ) {
1831 KMP_FATAL( LockUnsettingSetByAnother, func );
1833 return __kmp_release_nested_queuing_lock( lck, gtid );
1837 __kmp_init_nested_queuing_lock( kmp_queuing_lock_t * lck )
1839 __kmp_init_queuing_lock( lck );
1840 lck->lk.depth_locked = 0;
1844 __kmp_init_nested_queuing_lock_with_checks( kmp_queuing_lock_t * lck )
1846 __kmp_init_nested_queuing_lock( lck );
1850 __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck )
1852 __kmp_destroy_queuing_lock( lck );
1853 lck->lk.depth_locked = 0;
1857 __kmp_destroy_nested_queuing_lock_with_checks( kmp_queuing_lock_t *lck )
1859 char const *
const func =
"omp_destroy_nest_lock";
1860 if ( lck->lk.initialized != lck ) {
1861 KMP_FATAL( LockIsUninitialized, func );
1863 if ( ! __kmp_is_queuing_lock_nestable( lck ) ) {
1864 KMP_FATAL( LockSimpleUsedAsNestable, func );
1866 if ( __kmp_get_queuing_lock_owner( lck ) != -1 ) {
1867 KMP_FATAL( LockStillOwned, func );
1869 __kmp_destroy_nested_queuing_lock( lck );
1878 __kmp_is_queuing_lock_initialized( kmp_queuing_lock_t *lck )
1880 return lck == lck->lk.initialized;
1884 __kmp_get_queuing_lock_location( kmp_queuing_lock_t *lck )
1886 return lck->lk.location;
1890 __kmp_set_queuing_lock_location( kmp_queuing_lock_t *lck,
const ident_t *loc )
1892 lck->lk.location = loc;
1895 static kmp_lock_flags_t
1896 __kmp_get_queuing_lock_flags( kmp_queuing_lock_t *lck )
1898 return lck->lk.flags;
1902 __kmp_set_queuing_lock_flags( kmp_queuing_lock_t *lck, kmp_lock_flags_t flags )
1904 lck->lk.flags = flags;
1907 #if KMP_USE_ADAPTIVE_LOCKS 1913 #if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 1915 #include <immintrin.h> 1916 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1921 #define _XBEGIN_STARTED (~0u) 1922 #define _XABORT_EXPLICIT (1 << 0) 1923 #define _XABORT_RETRY (1 << 1) 1924 #define _XABORT_CONFLICT (1 << 2) 1925 #define _XABORT_CAPACITY (1 << 3) 1926 #define _XABORT_DEBUG (1 << 4) 1927 #define _XABORT_NESTED (1 << 5) 1928 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF)) 1931 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT) 1933 #define STRINGIZE_INTERNAL(arg) #arg 1934 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg) 1942 static __inline
int _xbegin()
1971 #endif // KMP_ARCH_X86_64 1981 __asm__
volatile (
"1: .byte 0xC7; .byte 0xF8;\n" 1984 "1: movl %%eax,%0\n" 1986 :
"+r"(res)::
"memory",
"%eax");
1987 #endif // KMP_OS_WINDOWS 1994 static __inline
void _xend()
2003 __asm__
volatile (
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
2013 #define _xabort(ARG) \ 2018 #define _xabort(ARG) \ 2019 __asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory"); 2022 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300 2027 #if KMP_DEBUG_ADAPTIVE_LOCKS 2032 static kmp_adaptive_lock_statistics_t destroyedStats;
2035 static kmp_adaptive_lock_info_t liveLocks;
2038 static kmp_bootstrap_lock_t chain_lock;
2042 __kmp_init_speculative_stats()
2044 kmp_adaptive_lock_info_t *lck = &liveLocks;
2046 memset( (
void * ) & ( lck->stats ), 0,
sizeof( lck->stats ) );
2047 lck->stats.next = lck;
2048 lck->stats.prev = lck;
2050 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2051 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2053 __kmp_init_bootstrap_lock( &chain_lock );
2059 __kmp_remember_lock( kmp_adaptive_lock_info_t * lck )
2061 __kmp_acquire_bootstrap_lock( &chain_lock );
2063 lck->stats.next = liveLocks.stats.next;
2064 lck->stats.prev = &liveLocks;
2066 liveLocks.stats.next = lck;
2067 lck->stats.next->stats.prev = lck;
2069 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2070 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2072 __kmp_release_bootstrap_lock( &chain_lock );
2076 __kmp_forget_lock( kmp_adaptive_lock_info_t * lck )
2078 KMP_ASSERT( lck->stats.next->stats.prev == lck );
2079 KMP_ASSERT( lck->stats.prev->stats.next == lck );
2081 kmp_adaptive_lock_info_t * n = lck->stats.next;
2082 kmp_adaptive_lock_info_t * p = lck->stats.prev;
2089 __kmp_zero_speculative_stats( kmp_adaptive_lock_info_t * lck )
2091 memset( (
void * )&lck->stats, 0,
sizeof( lck->stats ) );
2092 __kmp_remember_lock( lck );
2096 __kmp_add_stats( kmp_adaptive_lock_statistics_t * t, kmp_adaptive_lock_info_t * lck )
2098 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
2100 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
2101 t->successfulSpeculations += s->successfulSpeculations;
2102 t->hardFailedSpeculations += s->hardFailedSpeculations;
2103 t->softFailedSpeculations += s->softFailedSpeculations;
2104 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
2105 t->lemmingYields += s->lemmingYields;
2109 __kmp_accumulate_speculative_stats( kmp_adaptive_lock_info_t * lck)
2111 kmp_adaptive_lock_statistics_t *t = &destroyedStats;
2113 __kmp_acquire_bootstrap_lock( &chain_lock );
2115 __kmp_add_stats( &destroyedStats, lck );
2116 __kmp_forget_lock( lck );
2118 __kmp_release_bootstrap_lock( &chain_lock );
2122 percent (kmp_uint32 count, kmp_uint32 total)
2124 return (total == 0) ? 0.0: (100.0 * count)/total;
2128 FILE * __kmp_open_stats_file()
2130 if (strcmp (__kmp_speculative_statsfile,
"-") == 0)
2133 size_t buffLen = KMP_STRLEN( __kmp_speculative_statsfile ) + 20;
2134 char buffer[buffLen];
2135 KMP_SNPRINTF (&buffer[0], buffLen, __kmp_speculative_statsfile,
2136 (kmp_int32)getpid());
2137 FILE * result = fopen(&buffer[0],
"w");
2140 return result ? result : stdout;
2144 __kmp_print_speculative_stats()
2146 if (__kmp_user_lock_kind != lk_adaptive)
2149 FILE * statsFile = __kmp_open_stats_file();
2151 kmp_adaptive_lock_statistics_t total = destroyedStats;
2152 kmp_adaptive_lock_info_t *lck;
2154 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
2155 __kmp_add_stats( &total, lck );
2157 kmp_adaptive_lock_statistics_t *t = &total;
2158 kmp_uint32 totalSections = t->nonSpeculativeAcquires + t->successfulSpeculations;
2159 kmp_uint32 totalSpeculations = t->successfulSpeculations + t->hardFailedSpeculations +
2160 t->softFailedSpeculations;
2162 fprintf ( statsFile,
"Speculative lock statistics (all approximate!)\n");
2163 fprintf ( statsFile,
" Lock parameters: \n" 2164 " max_soft_retries : %10d\n" 2165 " max_badness : %10d\n",
2166 __kmp_adaptive_backoff_params.max_soft_retries,
2167 __kmp_adaptive_backoff_params.max_badness);
2168 fprintf( statsFile,
" Non-speculative acquire attempts : %10d\n", t->nonSpeculativeAcquireAttempts );
2169 fprintf( statsFile,
" Total critical sections : %10d\n", totalSections );
2170 fprintf( statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
2171 t->successfulSpeculations, percent( t->successfulSpeculations, totalSections ) );
2172 fprintf( statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
2173 t->nonSpeculativeAcquires, percent( t->nonSpeculativeAcquires, totalSections ) );
2174 fprintf( statsFile,
" Lemming yields : %10d\n\n", t->lemmingYields );
2176 fprintf( statsFile,
" Speculative acquire attempts : %10d\n", totalSpeculations );
2177 fprintf( statsFile,
" Successes : %10d (%5.1f%%)\n",
2178 t->successfulSpeculations, percent( t->successfulSpeculations, totalSpeculations ) );
2179 fprintf( statsFile,
" Soft failures : %10d (%5.1f%%)\n",
2180 t->softFailedSpeculations, percent( t->softFailedSpeculations, totalSpeculations ) );
2181 fprintf( statsFile,
" Hard failures : %10d (%5.1f%%)\n",
2182 t->hardFailedSpeculations, percent( t->hardFailedSpeculations, totalSpeculations ) );
2184 if (statsFile != stdout)
2185 fclose( statsFile );
2188 # define KMP_INC_STAT(lck,stat) ( lck->lk.adaptive.stats.stat++ ) 2190 # define KMP_INC_STAT(lck,stat) 2192 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 2195 __kmp_is_unlocked_queuing_lock( kmp_queuing_lock_t *lck )
2199 bool res = lck->lk.head_id == 0;
2203 #if KMP_COMPILER_ICC 2206 __sync_synchronize();
2213 static __inline
void 2214 __kmp_update_badness_after_success( kmp_adaptive_lock_t *lck )
2217 lck->lk.adaptive.badness = 0;
2218 KMP_INC_STAT(lck,successfulSpeculations);
2222 static __inline
void 2223 __kmp_step_badness( kmp_adaptive_lock_t *lck )
2225 kmp_uint32 newBadness = ( lck->lk.adaptive.badness << 1 ) | 1;
2226 if ( newBadness > lck->lk.adaptive.max_badness) {
2229 lck->lk.adaptive.badness = newBadness;
2235 __kmp_should_speculate( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2237 kmp_uint32 badness = lck->lk.adaptive.badness;
2238 kmp_uint32 attempts= lck->lk.adaptive.acquire_attempts;
2239 int res = (attempts & badness) == 0;
2247 __kmp_test_adaptive_lock_only( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2249 int retries = lck->lk.adaptive.max_soft_retries;
2257 kmp_uint32 status = _xbegin();
2262 if (status == _XBEGIN_STARTED )
2268 if (! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2274 KMP_ASSERT2( 0,
"should not get here" );
2279 if ( status & SOFT_ABORT_MASK)
2281 KMP_INC_STAT(lck,softFailedSpeculations);
2286 KMP_INC_STAT(lck,hardFailedSpeculations);
2291 }
while( retries-- );
2295 __kmp_step_badness( lck );
2303 __kmp_test_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2306 if ( __kmp_should_speculate( lck, gtid ) && __kmp_test_adaptive_lock_only( lck, gtid ) )
2311 lck->lk.adaptive.acquire_attempts++;
2314 if ( __kmp_test_queuing_lock( GET_QLK_PTR(lck), gtid ) )
2316 KMP_INC_STAT(lck,nonSpeculativeAcquires);
2326 __kmp_test_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2328 char const *
const func =
"omp_test_lock";
2329 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2330 KMP_FATAL( LockIsUninitialized, func );
2333 int retval = __kmp_test_adaptive_lock( lck, gtid );
2336 lck->lk.qlk.owner_id = gtid + 1;
2356 __kmp_acquire_adaptive_lock( kmp_adaptive_lock_t * lck, kmp_int32 gtid )
2358 if ( __kmp_should_speculate( lck, gtid ) )
2360 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2362 if ( __kmp_test_adaptive_lock_only( lck , gtid ) )
2374 while ( ! __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2376 KMP_INC_STAT(lck,lemmingYields);
2380 if ( __kmp_test_adaptive_lock_only( lck, gtid ) )
2387 lck->lk.adaptive.acquire_attempts++;
2389 __kmp_acquire_queuing_lock_timed_template<FALSE>( GET_QLK_PTR(lck), gtid );
2391 KMP_INC_STAT(lck,nonSpeculativeAcquires );
2392 ANNOTATE_QUEUING_ACQUIRED(lck);
2396 __kmp_acquire_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2398 char const *
const func =
"omp_set_lock";
2399 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2400 KMP_FATAL( LockIsUninitialized, func );
2402 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == gtid ) {
2403 KMP_FATAL( LockIsAlreadyOwned, func );
2406 __kmp_acquire_adaptive_lock( lck, gtid );
2408 lck->lk.qlk.owner_id = gtid + 1;
2412 __kmp_release_adaptive_lock( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2414 if ( __kmp_is_unlocked_queuing_lock( GET_QLK_PTR(lck) ) )
2419 __kmp_update_badness_after_success( lck );
2424 __kmp_release_queuing_lock( GET_QLK_PTR(lck), gtid );
2426 return KMP_LOCK_RELEASED;
2430 __kmp_release_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck, kmp_int32 gtid )
2432 char const *
const func =
"omp_unset_lock";
2434 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2435 KMP_FATAL( LockIsUninitialized, func );
2437 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) == -1 ) {
2438 KMP_FATAL( LockUnsettingFree, func );
2440 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != gtid ) {
2441 KMP_FATAL( LockUnsettingSetByAnother, func );
2443 lck->lk.qlk.owner_id = 0;
2444 __kmp_release_adaptive_lock( lck, gtid );
2445 return KMP_LOCK_RELEASED;
2449 __kmp_init_adaptive_lock( kmp_adaptive_lock_t *lck )
2451 __kmp_init_queuing_lock( GET_QLK_PTR(lck) );
2452 lck->lk.adaptive.badness = 0;
2453 lck->lk.adaptive.acquire_attempts = 0;
2454 lck->lk.adaptive.max_soft_retries = __kmp_adaptive_backoff_params.max_soft_retries;
2455 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2456 #if KMP_DEBUG_ADAPTIVE_LOCKS 2457 __kmp_zero_speculative_stats( &lck->lk.adaptive );
2459 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2463 __kmp_init_adaptive_lock_with_checks( kmp_adaptive_lock_t * lck )
2465 __kmp_init_adaptive_lock( lck );
2469 __kmp_destroy_adaptive_lock( kmp_adaptive_lock_t *lck )
2471 #if KMP_DEBUG_ADAPTIVE_LOCKS 2472 __kmp_accumulate_speculative_stats( &lck->lk.adaptive );
2474 __kmp_destroy_queuing_lock (GET_QLK_PTR(lck));
2479 __kmp_destroy_adaptive_lock_with_checks( kmp_adaptive_lock_t *lck )
2481 char const *
const func =
"omp_destroy_lock";
2482 if ( lck->lk.qlk.initialized != GET_QLK_PTR(lck) ) {
2483 KMP_FATAL( LockIsUninitialized, func );
2485 if ( __kmp_get_queuing_lock_owner( GET_QLK_PTR(lck) ) != -1 ) {
2486 KMP_FATAL( LockStillOwned, func );
2488 __kmp_destroy_adaptive_lock( lck );
2492 #endif // KMP_USE_ADAPTIVE_LOCKS 2500 __kmp_get_drdpa_lock_owner( kmp_drdpa_lock_t *lck )
2502 return TCR_4( lck->lk.owner_id ) - 1;
2506 __kmp_is_drdpa_lock_nestable( kmp_drdpa_lock_t *lck )
2508 return lck->lk.depth_locked != -1;
2511 __forceinline
static int 2512 __kmp_acquire_drdpa_lock_timed_template( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2514 kmp_uint64 ticket = KMP_TEST_THEN_INC64((kmp_int64 *)&lck->lk.next_ticket);
2515 kmp_uint64 mask = TCR_8(lck->lk.mask);
2516 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2517 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2518 TCR_PTR(lck->lk.polls);
2520 #ifdef USE_LOCK_PROFILE 2521 if (TCR_8(polls[ticket & mask].poll) != ticket)
2522 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2538 KMP_FSYNC_PREPARE(lck);
2539 KMP_INIT_YIELD(spins);
2540 while (TCR_8(polls[ticket & mask].poll) < ticket) {
2545 KMP_YIELD(TCR_4(__kmp_nth)
2546 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2547 KMP_YIELD_SPIN(spins);
2558 mask = TCR_8(lck->lk.mask);
2559 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2560 TCR_PTR(lck->lk.polls);
2566 KMP_FSYNC_ACQUIRED(lck);
2567 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2569 lck->lk.now_serving = ticket;
2578 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2579 __kmp_free((
void *)lck->lk.old_polls);
2580 lck->lk.old_polls = NULL;
2581 lck->lk.cleanup_ticket = 0;
2589 if (lck->lk.old_polls == NULL) {
2590 bool reconfigure =
false;
2591 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *old_polls = polls;
2592 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2594 if (TCR_4(__kmp_nth)
2595 > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2600 if (num_polls > 1) {
2602 num_polls = TCR_4(lck->lk.num_polls);
2605 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2606 __kmp_allocate(num_polls *
sizeof(*polls));
2607 polls[0].poll = ticket;
2616 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2617 if (num_waiting > num_polls) {
2618 kmp_uint32 old_num_polls = num_polls;
2621 mask = (mask << 1) | 1;
2623 }
while (num_polls <= num_waiting);
2631 polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2632 __kmp_allocate(num_polls *
sizeof(*polls));
2634 for (i = 0; i < old_num_polls; i++) {
2635 polls[i].poll = old_polls[i].poll;
2652 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring lock %p to %d polls\n",
2653 ticket, lck, num_polls));
2655 lck->lk.old_polls = old_polls;
2656 lck->lk.polls = polls;
2660 lck->lk.num_polls = num_polls;
2661 lck->lk.mask = mask;
2671 lck->lk.cleanup_ticket = TCR_8(lck->lk.next_ticket);
2674 return KMP_LOCK_ACQUIRED_FIRST;
2678 __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2680 int retval = __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2681 ANNOTATE_DRDPA_ACQUIRED(lck);
2686 __kmp_acquire_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2688 char const *
const func =
"omp_set_lock";
2689 if ( lck->lk.initialized != lck ) {
2690 KMP_FATAL( LockIsUninitialized, func );
2692 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2693 KMP_FATAL( LockNestableUsedAsSimple, func );
2695 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) ) {
2696 KMP_FATAL( LockIsAlreadyOwned, func );
2699 __kmp_acquire_drdpa_lock( lck, gtid );
2701 lck->lk.owner_id = gtid + 1;
2702 return KMP_LOCK_ACQUIRED_FIRST;
2706 __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2712 kmp_uint64 ticket = TCR_8(lck->lk.next_ticket);
2713 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2714 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2715 TCR_PTR(lck->lk.polls);
2716 kmp_uint64 mask = TCR_8(lck->lk.mask);
2717 if (TCR_8(polls[ticket & mask].poll) == ticket) {
2718 kmp_uint64 next_ticket = ticket + 1;
2719 if (KMP_COMPARE_AND_STORE_ACQ64((kmp_int64 *)&lck->lk.next_ticket,
2720 ticket, next_ticket)) {
2721 KMP_FSYNC_ACQUIRED(lck);
2722 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2724 lck->lk.now_serving = ticket;
2742 __kmp_test_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2744 char const *
const func =
"omp_test_lock";
2745 if ( lck->lk.initialized != lck ) {
2746 KMP_FATAL( LockIsUninitialized, func );
2748 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2749 KMP_FATAL( LockNestableUsedAsSimple, func );
2752 int retval = __kmp_test_drdpa_lock( lck, gtid );
2755 lck->lk.owner_id = gtid + 1;
2761 __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2768 kmp_uint64 ticket = lck->lk.now_serving + 1;
2769 volatile struct kmp_base_drdpa_lock::kmp_lock_poll *polls
2770 = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2771 TCR_PTR(lck->lk.polls);
2772 kmp_uint64 mask = TCR_8(lck->lk.mask);
2773 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2775 KMP_FSYNC_RELEASING(lck);
2776 ANNOTATE_DRDPA_RELEASED(lck);
2777 KMP_ST_REL64(&(polls[ticket & mask].poll), ticket);
2778 return KMP_LOCK_RELEASED;
2782 __kmp_release_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2784 char const *
const func =
"omp_unset_lock";
2786 if ( lck->lk.initialized != lck ) {
2787 KMP_FATAL( LockIsUninitialized, func );
2789 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2790 KMP_FATAL( LockNestableUsedAsSimple, func );
2792 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2793 KMP_FATAL( LockUnsettingFree, func );
2795 if ( ( gtid >= 0 ) && ( __kmp_get_drdpa_lock_owner( lck ) >= 0 )
2796 && ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) ) {
2797 KMP_FATAL( LockUnsettingSetByAnother, func );
2799 lck->lk.owner_id = 0;
2800 return __kmp_release_drdpa_lock( lck, gtid );
2804 __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck )
2806 lck->lk.location = NULL;
2808 lck->lk.num_polls = 1;
2809 lck->lk.polls = (
volatile struct kmp_base_drdpa_lock::kmp_lock_poll *)
2810 __kmp_allocate(lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2811 lck->lk.cleanup_ticket = 0;
2812 lck->lk.old_polls = NULL;
2813 lck->lk.next_ticket = 0;
2814 lck->lk.now_serving = 0;
2815 lck->lk.owner_id = 0;
2816 lck->lk.depth_locked = -1;
2817 lck->lk.initialized = lck;
2819 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2823 __kmp_init_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2825 __kmp_init_drdpa_lock( lck );
2829 __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck )
2831 lck->lk.initialized = NULL;
2832 lck->lk.location = NULL;
2833 if (lck->lk.polls != NULL) {
2834 __kmp_free((
void *)lck->lk.polls);
2835 lck->lk.polls = NULL;
2837 if (lck->lk.old_polls != NULL) {
2838 __kmp_free((
void *)lck->lk.old_polls);
2839 lck->lk.old_polls = NULL;
2842 lck->lk.num_polls = 0;
2843 lck->lk.cleanup_ticket = 0;
2844 lck->lk.next_ticket = 0;
2845 lck->lk.now_serving = 0;
2846 lck->lk.owner_id = 0;
2847 lck->lk.depth_locked = -1;
2851 __kmp_destroy_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2853 char const *
const func =
"omp_destroy_lock";
2854 if ( lck->lk.initialized != lck ) {
2855 KMP_FATAL( LockIsUninitialized, func );
2857 if ( __kmp_is_drdpa_lock_nestable( lck ) ) {
2858 KMP_FATAL( LockNestableUsedAsSimple, func );
2860 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
2861 KMP_FATAL( LockStillOwned, func );
2863 __kmp_destroy_drdpa_lock( lck );
2872 __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2874 KMP_DEBUG_ASSERT( gtid >= 0 );
2876 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2877 lck->lk.depth_locked += 1;
2878 return KMP_LOCK_ACQUIRED_NEXT;
2881 __kmp_acquire_drdpa_lock_timed_template( lck, gtid );
2882 ANNOTATE_DRDPA_ACQUIRED(lck);
2884 lck->lk.depth_locked = 1;
2886 lck->lk.owner_id = gtid + 1;
2887 return KMP_LOCK_ACQUIRED_FIRST;
2892 __kmp_acquire_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2894 char const *
const func =
"omp_set_nest_lock";
2895 if ( lck->lk.initialized != lck ) {
2896 KMP_FATAL( LockIsUninitialized, func );
2898 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2899 KMP_FATAL( LockSimpleUsedAsNestable, func );
2901 __kmp_acquire_nested_drdpa_lock( lck, gtid );
2905 __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2909 KMP_DEBUG_ASSERT( gtid >= 0 );
2911 if ( __kmp_get_drdpa_lock_owner( lck ) == gtid ) {
2912 retval = ++lck->lk.depth_locked;
2914 else if ( !__kmp_test_drdpa_lock( lck, gtid ) ) {
2919 retval = lck->lk.depth_locked = 1;
2921 lck->lk.owner_id = gtid + 1;
2927 __kmp_test_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2929 char const *
const func =
"omp_test_nest_lock";
2930 if ( lck->lk.initialized != lck ) {
2931 KMP_FATAL( LockIsUninitialized, func );
2933 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2934 KMP_FATAL( LockSimpleUsedAsNestable, func );
2936 return __kmp_test_nested_drdpa_lock( lck, gtid );
2940 __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2942 KMP_DEBUG_ASSERT( gtid >= 0 );
2945 if ( --(lck->lk.depth_locked) == 0 ) {
2947 lck->lk.owner_id = 0;
2948 __kmp_release_drdpa_lock( lck, gtid );
2949 return KMP_LOCK_RELEASED;
2951 return KMP_LOCK_STILL_HELD;
2955 __kmp_release_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck, kmp_int32 gtid )
2957 char const *
const func =
"omp_unset_nest_lock";
2959 if ( lck->lk.initialized != lck ) {
2960 KMP_FATAL( LockIsUninitialized, func );
2962 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
2963 KMP_FATAL( LockSimpleUsedAsNestable, func );
2965 if ( __kmp_get_drdpa_lock_owner( lck ) == -1 ) {
2966 KMP_FATAL( LockUnsettingFree, func );
2968 if ( __kmp_get_drdpa_lock_owner( lck ) != gtid ) {
2969 KMP_FATAL( LockUnsettingSetByAnother, func );
2971 return __kmp_release_nested_drdpa_lock( lck, gtid );
2975 __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t * lck )
2977 __kmp_init_drdpa_lock( lck );
2978 lck->lk.depth_locked = 0;
2982 __kmp_init_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t * lck )
2984 __kmp_init_nested_drdpa_lock( lck );
2988 __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck )
2990 __kmp_destroy_drdpa_lock( lck );
2991 lck->lk.depth_locked = 0;
2995 __kmp_destroy_nested_drdpa_lock_with_checks( kmp_drdpa_lock_t *lck )
2997 char const *
const func =
"omp_destroy_nest_lock";
2998 if ( lck->lk.initialized != lck ) {
2999 KMP_FATAL( LockIsUninitialized, func );
3001 if ( ! __kmp_is_drdpa_lock_nestable( lck ) ) {
3002 KMP_FATAL( LockSimpleUsedAsNestable, func );
3004 if ( __kmp_get_drdpa_lock_owner( lck ) != -1 ) {
3005 KMP_FATAL( LockStillOwned, func );
3007 __kmp_destroy_nested_drdpa_lock( lck );
3016 __kmp_is_drdpa_lock_initialized( kmp_drdpa_lock_t *lck )
3018 return lck == lck->lk.initialized;
3022 __kmp_get_drdpa_lock_location( kmp_drdpa_lock_t *lck )
3024 return lck->lk.location;
3028 __kmp_set_drdpa_lock_location( kmp_drdpa_lock_t *lck,
const ident_t *loc )
3030 lck->lk.location = loc;
3033 static kmp_lock_flags_t
3034 __kmp_get_drdpa_lock_flags( kmp_drdpa_lock_t *lck )
3036 return lck->lk.flags;
3040 __kmp_set_drdpa_lock_flags( kmp_drdpa_lock_t *lck, kmp_lock_flags_t flags )
3042 lck->lk.flags = flags;
3046 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 3047 # define __kmp_tsc() __kmp_hardware_timestamp() 3049 kmp_backoff_t __kmp_spin_backoff_params = { 1, 4096, 100 };
3052 extern kmp_uint64 __kmp_now_nsec();
3053 kmp_backoff_t __kmp_spin_backoff_params = { 1, 256, 100 };
3054 # define __kmp_tsc() __kmp_now_nsec() 3065 static inline bool before(kmp_uint64 a, kmp_uint64 b)
3067 return ((kmp_int64)b - (kmp_int64)a) > 0;
3072 __kmp_spin_backoff(kmp_backoff_t *boff)
3076 for (i = boff->step; i > 0; i--) {
3077 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
3080 }
while (before(__kmp_tsc(), goal));
3082 boff->step = (boff->step<<1 | 1) & (boff->max_backoff-1);
3085 #if KMP_USE_DYNAMIC_LOCK 3088 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq)
3090 TCW_4(*lck, KMP_GET_D_TAG(seq));
3091 KA_TRACE(20, (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
3097 #define HLE_ACQUIRE ".byte 0xf2;" 3098 #define HLE_RELEASE ".byte 0xf3;" 3100 static inline kmp_uint32
3101 swap4(kmp_uint32
volatile *p, kmp_uint32 v)
3103 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" 3111 __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
3117 __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3120 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
3123 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
3124 for (
int i = delay; i != 0; --i)
3126 delay = ((delay << 1) | 1) & 7;
3128 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
3133 __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3135 __kmp_acquire_hle_lock(lck, gtid);
3139 __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3141 __asm__
volatile(HLE_RELEASE
"movl %1,%0" 3143 :
"r"(KMP_LOCK_FREE(hle))
3145 return KMP_LOCK_RELEASED;
3149 __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3151 return __kmp_release_hle_lock(lck, gtid);
3155 __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3157 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
3161 __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
3163 return __kmp_test_hle_lock(lck, gtid);
3167 __kmp_init_rtm_lock(kmp_queuing_lock_t *lck)
3169 __kmp_init_queuing_lock(lck);
3173 __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck)
3175 __kmp_destroy_queuing_lock(lck);
3179 __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3181 unsigned retries=3, status;
3184 if (status == _XBEGIN_STARTED) {
3185 if (__kmp_is_unlocked_queuing_lock(lck))
3189 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
3191 while (! __kmp_is_unlocked_queuing_lock(lck))
3194 else if (!(status & _XABORT_RETRY))
3196 }
while (retries--);
3199 __kmp_acquire_queuing_lock(lck, gtid);
3203 __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3205 __kmp_acquire_rtm_lock(lck, gtid);
3209 __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3211 if (__kmp_is_unlocked_queuing_lock(lck)) {
3217 __kmp_release_queuing_lock(lck, gtid);
3219 return KMP_LOCK_RELEASED;
3223 __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3225 return __kmp_release_rtm_lock(lck, gtid);
3229 __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3231 unsigned retries=3, status;
3234 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
3237 if (!(status & _XABORT_RETRY))
3239 }
while (retries--);
3241 return (__kmp_is_unlocked_queuing_lock(lck))? 1: 0;
3245 __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck, kmp_int32 gtid)
3247 return __kmp_test_rtm_lock(lck, gtid);
3250 #endif // KMP_USE_TSX 3253 static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
3254 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
3255 static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3256 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3257 static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
3258 static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3259 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3260 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
3268 #define expand(l, op) 0,__kmp_init_direct_lock, 3269 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
3270 = { __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init) };
3274 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock, 3275 void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *)
3276 = { __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy) };
3280 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 3281 static void (*direct_set[])(kmp_dyna_lock_t *, kmp_int32)
3282 = { __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
3284 #define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 3285 static void (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32)
3286 = { __kmp_set_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
3290 #define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock, 3291 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32)
3292 = { __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release) };
3293 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32)
3294 = { __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test) };
3296 #define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks, 3297 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32)
3298 = { __kmp_unset_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, release) };
3299 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32)
3300 = { __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test) };
3304 void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
3305 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
3306 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
3311 #define expand(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock, 3312 void (*__kmp_indirect_init[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, init) };
3313 void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, destroy) };
3317 #define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3318 static void (*indirect_set[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
3320 #define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3321 static void (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
3325 #define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock, 3326 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
3327 static int (*indirect_test[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
3329 #define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks, 3330 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
3331 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
3335 void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3336 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3337 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3340 kmp_indirect_lock_table_t __kmp_i_lock_table;
3343 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = { 0 };
3346 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
const ident_t *) = { 0 };
3347 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t) = { 0 };
3348 const ident_t * (*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3349 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p) = { 0 };
3352 static kmp_indirect_lock_t * __kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = { 0 };
3358 kmp_indirect_lock_t *
3359 __kmp_allocate_indirect_lock(
void **user_lock, kmp_int32 gtid, kmp_indirect_locktag_t tag)
3361 kmp_indirect_lock_t *lck;
3362 kmp_lock_index_t idx;
3364 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3366 if (__kmp_indirect_lock_pool[tag] != NULL) {
3368 lck = __kmp_indirect_lock_pool[tag];
3369 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3370 idx = lck->lock->pool.index;
3371 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3372 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n", lck));
3374 idx = __kmp_i_lock_table.next;
3376 if (idx == __kmp_i_lock_table.size) {
3378 int row = __kmp_i_lock_table.size/KMP_I_LOCK_CHUNK;
3379 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3380 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(2*row*
sizeof(kmp_indirect_lock_t *));
3381 KMP_MEMCPY(__kmp_i_lock_table.table, old_table, row*
sizeof(kmp_indirect_lock_t *));
3382 __kmp_free(old_table);
3384 for (
int i = row; i < 2*row; ++i)
3385 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)
3386 __kmp_allocate(KMP_I_LOCK_CHUNK*
sizeof(kmp_indirect_lock_t));
3387 __kmp_i_lock_table.size = 2*idx;
3389 __kmp_i_lock_table.next++;
3390 lck = KMP_GET_I_LOCK(idx);
3392 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3393 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3396 __kmp_release_lock(&__kmp_global_lock, gtid);
3400 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3401 *((kmp_lock_index_t *)user_lock) = idx << 1;
3403 *((kmp_indirect_lock_t **)user_lock) = lck;
3410 static __forceinline
3411 kmp_indirect_lock_t *
3412 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func)
3414 if (__kmp_env_consistency_check) {
3415 kmp_indirect_lock_t *lck = NULL;
3416 if (user_lock == NULL) {
3417 KMP_FATAL(LockIsUninitialized, func);
3419 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3420 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3421 if (idx >= __kmp_i_lock_table.size) {
3422 KMP_FATAL(LockIsUninitialized, func);
3424 lck = KMP_GET_I_LOCK(idx);
3426 lck = *((kmp_indirect_lock_t **)user_lock);
3429 KMP_FATAL(LockIsUninitialized, func);
3433 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3434 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3436 return *((kmp_indirect_lock_t **)user_lock);
3442 __kmp_init_indirect_lock(kmp_dyna_lock_t * lock, kmp_dyna_lockseq_t seq)
3444 #if KMP_USE_ADAPTIVE_LOCKS 3445 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3446 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3447 seq = lockseq_queuing;
3451 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3452 seq = lockseq_queuing;
3455 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3456 kmp_indirect_lock_t *l = __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3457 KMP_I_LOCK_FUNC(l, init)(l->lock);
3458 KA_TRACE(20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n", seq));
3462 __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock)
3464 kmp_uint32 gtid = __kmp_entry_gtid();
3465 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3466 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3467 kmp_indirect_locktag_t tag = l->type;
3469 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3472 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3473 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3474 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3476 __kmp_indirect_lock_pool[tag] = l;
3478 __kmp_release_lock(&__kmp_global_lock, gtid);
3482 __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3484 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3485 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3489 __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3491 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3492 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3496 __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3498 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3499 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3503 __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3505 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3506 KMP_I_LOCK_FUNC(l,
set)(l->lock, gtid);
3510 __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3512 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3513 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3517 __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
3519 kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3520 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3523 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3527 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq)
3531 case lockseq_nested_tas:
3532 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3535 case lockseq_nested_futex:
3536 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3538 case lockseq_ticket:
3539 case lockseq_nested_ticket:
3540 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3541 case lockseq_queuing:
3542 case lockseq_nested_queuing:
3543 #if KMP_USE_ADAPTIVE_LOCKS 3544 case lockseq_adaptive:
3546 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3548 case lockseq_nested_drdpa:
3549 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3557 __kmp_init_dynamic_user_locks()
3560 if (__kmp_env_consistency_check) {
3561 __kmp_direct_set = direct_set_check;
3562 __kmp_direct_unset = direct_unset_check;
3563 __kmp_direct_test = direct_test_check;
3564 __kmp_indirect_set = indirect_set_check;
3565 __kmp_indirect_unset = indirect_unset_check;
3566 __kmp_indirect_test = indirect_test_check;
3569 __kmp_direct_set = direct_set;
3570 __kmp_direct_unset = direct_unset;
3571 __kmp_direct_test = direct_test;
3572 __kmp_indirect_set = indirect_set;
3573 __kmp_indirect_unset = indirect_unset;
3574 __kmp_indirect_test = indirect_test;
3580 if (__kmp_init_user_locks)
3584 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3585 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3586 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)
3587 __kmp_allocate(KMP_I_LOCK_CHUNK*
sizeof(kmp_indirect_lock_t));
3588 __kmp_i_lock_table.next = 0;
3591 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3592 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3593 #if KMP_USE_ADAPTIVE_LOCKS 3594 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3596 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3598 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3600 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3602 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3604 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3605 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3606 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3609 #define fill_jumps(table, expand, sep) { \ 3610 table[locktag##sep##ticket] = expand(ticket); \ 3611 table[locktag##sep##queuing] = expand(queuing); \ 3612 table[locktag##sep##drdpa] = expand(drdpa); \ 3615 #if KMP_USE_ADAPTIVE_LOCKS 3616 # define fill_table(table, expand) { \ 3617 fill_jumps(table, expand, _); \ 3618 table[locktag_adaptive] = expand(queuing); \ 3619 fill_jumps(table, expand, _nested_); \ 3622 # define fill_table(table, expand) { \ 3623 fill_jumps(table, expand, _); \ 3624 fill_jumps(table, expand, _nested_); \ 3626 #endif // KMP_USE_ADAPTIVE_LOCKS 3628 #define expand(l) (void (*)(kmp_user_lock_p, const ident_t *))__kmp_set_##l##_lock_location 3629 fill_table(__kmp_indirect_set_location, expand);
3631 #define expand(l) (void (*)(kmp_user_lock_p, kmp_lock_flags_t))__kmp_set_##l##_lock_flags 3632 fill_table(__kmp_indirect_set_flags, expand);
3634 #define expand(l) (const ident_t * (*)(kmp_user_lock_p))__kmp_get_##l##_lock_location 3635 fill_table(__kmp_indirect_get_location, expand);
3637 #define expand(l) (kmp_lock_flags_t (*)(kmp_user_lock_p))__kmp_get_##l##_lock_flags 3638 fill_table(__kmp_indirect_get_flags, expand);
3641 __kmp_init_user_locks = TRUE;
3646 __kmp_cleanup_indirect_user_locks()
3652 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3653 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3655 kmp_indirect_lock_t *ll = l;
3656 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3657 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n", ll));
3658 __kmp_free(ll->lock);
3661 __kmp_indirect_lock_pool[k] = NULL;
3664 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3665 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3666 if (l->lock != NULL) {
3668 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3669 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n", l));
3670 __kmp_free(l->lock);
3674 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3675 __kmp_free(__kmp_i_lock_table.table[i]);
3676 __kmp_free(__kmp_i_lock_table.table);
3678 __kmp_init_user_locks = FALSE;
3681 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3682 int __kmp_num_locks_in_block = 1;
3684 #else // KMP_USE_DYNAMIC_LOCK 3693 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3695 size_t __kmp_base_user_lock_size = 0;
3696 size_t __kmp_user_lock_size = 0;
3698 kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck ) = NULL;
3699 int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3701 int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3702 int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3703 void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3704 void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck ) = NULL;
3705 void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3706 int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3708 int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3709 int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid ) = NULL;
3710 void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3711 void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck ) = NULL;
3713 int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck ) = NULL;
3714 const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck ) = NULL;
3715 void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck,
const ident_t *loc ) = NULL;
3716 kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck ) = NULL;
3717 void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags ) = NULL;
3719 void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind )
3721 switch ( user_lock_kind ) {
3727 __kmp_base_user_lock_size =
sizeof( kmp_base_tas_lock_t );
3728 __kmp_user_lock_size =
sizeof( kmp_tas_lock_t );
3730 __kmp_get_user_lock_owner_ =
3731 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3732 ( &__kmp_get_tas_lock_owner );
3734 if ( __kmp_env_consistency_check ) {
3735 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3736 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3739 KMP_BIND_USER_LOCK(tas);
3740 KMP_BIND_NESTED_USER_LOCK(tas);
3743 __kmp_destroy_user_lock_ =
3744 ( void ( * )( kmp_user_lock_p ) )
3745 ( &__kmp_destroy_tas_lock );
3747 __kmp_is_user_lock_initialized_ =
3748 ( int ( * )( kmp_user_lock_p ) ) NULL;
3750 __kmp_get_user_lock_location_ =
3751 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3753 __kmp_set_user_lock_location_ =
3754 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3756 __kmp_get_user_lock_flags_ =
3757 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3759 __kmp_set_user_lock_flags_ =
3760 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3767 __kmp_base_user_lock_size =
sizeof( kmp_base_futex_lock_t );
3768 __kmp_user_lock_size =
sizeof( kmp_futex_lock_t );
3770 __kmp_get_user_lock_owner_ =
3771 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3772 ( &__kmp_get_futex_lock_owner );
3774 if ( __kmp_env_consistency_check ) {
3775 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3776 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3779 KMP_BIND_USER_LOCK(futex);
3780 KMP_BIND_NESTED_USER_LOCK(futex);
3783 __kmp_destroy_user_lock_ =
3784 ( void ( * )( kmp_user_lock_p ) )
3785 ( &__kmp_destroy_futex_lock );
3787 __kmp_is_user_lock_initialized_ =
3788 ( int ( * )( kmp_user_lock_p ) ) NULL;
3790 __kmp_get_user_lock_location_ =
3791 (
const ident_t * ( * )( kmp_user_lock_p ) ) NULL;
3793 __kmp_set_user_lock_location_ =
3794 ( void ( * )( kmp_user_lock_p,
const ident_t * ) ) NULL;
3796 __kmp_get_user_lock_flags_ =
3797 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) ) NULL;
3799 __kmp_set_user_lock_flags_ =
3800 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) ) NULL;
3804 #endif // KMP_USE_FUTEX 3807 __kmp_base_user_lock_size =
sizeof( kmp_base_ticket_lock_t );
3808 __kmp_user_lock_size =
sizeof( kmp_ticket_lock_t );
3810 __kmp_get_user_lock_owner_ =
3811 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3812 ( &__kmp_get_ticket_lock_owner );
3814 if ( __kmp_env_consistency_check ) {
3815 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3816 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3819 KMP_BIND_USER_LOCK(ticket);
3820 KMP_BIND_NESTED_USER_LOCK(ticket);
3823 __kmp_destroy_user_lock_ =
3824 ( void ( * )( kmp_user_lock_p ) )
3825 ( &__kmp_destroy_ticket_lock );
3827 __kmp_is_user_lock_initialized_ =
3828 ( int ( * )( kmp_user_lock_p ) )
3829 ( &__kmp_is_ticket_lock_initialized );
3831 __kmp_get_user_lock_location_ =
3832 (
const ident_t * ( * )( kmp_user_lock_p ) )
3833 ( &__kmp_get_ticket_lock_location );
3835 __kmp_set_user_lock_location_ =
3836 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3837 ( &__kmp_set_ticket_lock_location );
3839 __kmp_get_user_lock_flags_ =
3840 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3841 ( &__kmp_get_ticket_lock_flags );
3843 __kmp_set_user_lock_flags_ =
3844 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3845 ( &__kmp_set_ticket_lock_flags );
3850 __kmp_base_user_lock_size =
sizeof( kmp_base_queuing_lock_t );
3851 __kmp_user_lock_size =
sizeof( kmp_queuing_lock_t );
3853 __kmp_get_user_lock_owner_ =
3854 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3855 ( &__kmp_get_queuing_lock_owner );
3857 if ( __kmp_env_consistency_check ) {
3858 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3859 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3862 KMP_BIND_USER_LOCK(queuing);
3863 KMP_BIND_NESTED_USER_LOCK(queuing);
3866 __kmp_destroy_user_lock_ =
3867 ( void ( * )( kmp_user_lock_p ) )
3868 ( &__kmp_destroy_queuing_lock );
3870 __kmp_is_user_lock_initialized_ =
3871 ( int ( * )( kmp_user_lock_p ) )
3872 ( &__kmp_is_queuing_lock_initialized );
3874 __kmp_get_user_lock_location_ =
3875 (
const ident_t * ( * )( kmp_user_lock_p ) )
3876 ( &__kmp_get_queuing_lock_location );
3878 __kmp_set_user_lock_location_ =
3879 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3880 ( &__kmp_set_queuing_lock_location );
3882 __kmp_get_user_lock_flags_ =
3883 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3884 ( &__kmp_get_queuing_lock_flags );
3886 __kmp_set_user_lock_flags_ =
3887 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3888 ( &__kmp_set_queuing_lock_flags );
3892 #if KMP_USE_ADAPTIVE_LOCKS 3894 __kmp_base_user_lock_size =
sizeof( kmp_base_adaptive_lock_t );
3895 __kmp_user_lock_size =
sizeof( kmp_adaptive_lock_t );
3897 __kmp_get_user_lock_owner_ =
3898 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3899 ( &__kmp_get_queuing_lock_owner );
3901 if ( __kmp_env_consistency_check ) {
3902 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3905 KMP_BIND_USER_LOCK(adaptive);
3908 __kmp_destroy_user_lock_ =
3909 ( void ( * )( kmp_user_lock_p ) )
3910 ( &__kmp_destroy_adaptive_lock );
3912 __kmp_is_user_lock_initialized_ =
3913 ( int ( * )( kmp_user_lock_p ) )
3914 ( &__kmp_is_queuing_lock_initialized );
3916 __kmp_get_user_lock_location_ =
3917 (
const ident_t * ( * )( kmp_user_lock_p ) )
3918 ( &__kmp_get_queuing_lock_location );
3920 __kmp_set_user_lock_location_ =
3921 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3922 ( &__kmp_set_queuing_lock_location );
3924 __kmp_get_user_lock_flags_ =
3925 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3926 ( &__kmp_get_queuing_lock_flags );
3928 __kmp_set_user_lock_flags_ =
3929 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3930 ( &__kmp_set_queuing_lock_flags );
3934 #endif // KMP_USE_ADAPTIVE_LOCKS 3937 __kmp_base_user_lock_size =
sizeof( kmp_base_drdpa_lock_t );
3938 __kmp_user_lock_size =
sizeof( kmp_drdpa_lock_t );
3940 __kmp_get_user_lock_owner_ =
3941 ( kmp_int32 ( * )( kmp_user_lock_p ) )
3942 ( &__kmp_get_drdpa_lock_owner );
3944 if ( __kmp_env_consistency_check ) {
3945 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3946 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3949 KMP_BIND_USER_LOCK(drdpa);
3950 KMP_BIND_NESTED_USER_LOCK(drdpa);
3953 __kmp_destroy_user_lock_ =
3954 ( void ( * )( kmp_user_lock_p ) )
3955 ( &__kmp_destroy_drdpa_lock );
3957 __kmp_is_user_lock_initialized_ =
3958 ( int ( * )( kmp_user_lock_p ) )
3959 ( &__kmp_is_drdpa_lock_initialized );
3961 __kmp_get_user_lock_location_ =
3962 (
const ident_t * ( * )( kmp_user_lock_p ) )
3963 ( &__kmp_get_drdpa_lock_location );
3965 __kmp_set_user_lock_location_ =
3966 ( void ( * )( kmp_user_lock_p,
const ident_t * ) )
3967 ( &__kmp_set_drdpa_lock_location );
3969 __kmp_get_user_lock_flags_ =
3970 ( kmp_lock_flags_t ( * )( kmp_user_lock_p ) )
3971 ( &__kmp_get_drdpa_lock_flags );
3973 __kmp_set_user_lock_flags_ =
3974 ( void ( * )( kmp_user_lock_p, kmp_lock_flags_t ) )
3975 ( &__kmp_set_drdpa_lock_flags );
3985 kmp_lock_table_t __kmp_user_lock_table = { 1, 0, NULL };
3986 kmp_user_lock_p __kmp_lock_pool = NULL;
3989 kmp_block_of_locks* __kmp_lock_blocks = NULL;
3990 int __kmp_num_locks_in_block = 1;
3992 static kmp_lock_index_t
3993 __kmp_lock_table_insert( kmp_user_lock_p lck )
3996 kmp_lock_index_t index;
3997 if ( __kmp_user_lock_table.used >= __kmp_user_lock_table.allocated ) {
3998 kmp_lock_index_t size;
3999 kmp_user_lock_p *table;
4001 if ( __kmp_user_lock_table.allocated == 0 ) {
4005 size = __kmp_user_lock_table.allocated * 2;
4007 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof( kmp_user_lock_p ) * size );
4008 KMP_MEMCPY( table + 1, __kmp_user_lock_table.table + 1,
sizeof( kmp_user_lock_p ) * ( __kmp_user_lock_table.used - 1 ) );
4009 table[ 0 ] = (kmp_user_lock_p)__kmp_user_lock_table.table;
4014 __kmp_user_lock_table.table = table;
4015 __kmp_user_lock_table.allocated = size;
4017 KMP_DEBUG_ASSERT( __kmp_user_lock_table.used < __kmp_user_lock_table.allocated );
4018 index = __kmp_user_lock_table.used;
4019 __kmp_user_lock_table.table[ index ] = lck;
4020 ++ __kmp_user_lock_table.used;
4024 static kmp_user_lock_p
4025 __kmp_lock_block_allocate()
4028 static int last_index = 0;
4029 if ( ( last_index >= __kmp_num_locks_in_block )
4030 || ( __kmp_lock_blocks == NULL ) ) {
4034 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4035 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
4036 char* buffer = (
char*)__kmp_allocate( space_for_locks +
sizeof( kmp_block_of_locks ) );
4038 kmp_block_of_locks *new_block = (kmp_block_of_locks *)(& buffer[space_for_locks]);
4039 new_block->next_block = __kmp_lock_blocks;
4040 new_block->locks = (
void *)buffer;
4043 __kmp_lock_blocks = new_block;
4045 kmp_user_lock_p ret = (kmp_user_lock_p)(& ( ( (
char *)( __kmp_lock_blocks->locks ) )
4046 [ last_index * __kmp_user_lock_size ] ) );
4056 __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
4057 kmp_lock_flags_t flags )
4059 kmp_user_lock_p lck;
4060 kmp_lock_index_t index;
4061 KMP_DEBUG_ASSERT( user_lock );
4063 __kmp_acquire_lock( &__kmp_global_lock, gtid );
4065 if ( __kmp_lock_pool == NULL ) {
4070 ANNOTATE_IGNORE_WRITES_BEGIN();
4071 if ( __kmp_num_locks_in_block <= 1 ) {
4072 lck = (kmp_user_lock_p) __kmp_allocate( __kmp_user_lock_size );
4075 lck = __kmp_lock_block_allocate();
4077 ANNOTATE_IGNORE_WRITES_END();
4081 index = __kmp_lock_table_insert( lck );
4085 lck = __kmp_lock_pool;
4086 index = __kmp_lock_pool->pool.index;
4087 __kmp_lock_pool = __kmp_lock_pool->pool.next;
4094 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4095 * ( (kmp_lock_index_t *) user_lock ) = index;
4098 * ( (kmp_user_lock_p *) user_lock ) = lck;
4102 __kmp_set_user_lock_flags( lck, flags );
4104 __kmp_release_lock( & __kmp_global_lock, gtid );
4111 __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck )
4113 KMP_DEBUG_ASSERT( user_lock != NULL );
4114 KMP_DEBUG_ASSERT( lck != NULL );
4116 __kmp_acquire_lock( & __kmp_global_lock, gtid );
4118 lck->pool.next = __kmp_lock_pool;
4119 __kmp_lock_pool = lck;
4120 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4121 kmp_lock_index_t index = * ( (kmp_lock_index_t *) user_lock );
4122 KMP_DEBUG_ASSERT( 0 < index && index <= __kmp_user_lock_table.used );
4123 lck->pool.index = index;
4126 __kmp_release_lock( & __kmp_global_lock, gtid );
4130 __kmp_lookup_user_lock(
void **user_lock,
char const *func )
4132 kmp_user_lock_p lck = NULL;
4134 if ( __kmp_env_consistency_check ) {
4135 if ( user_lock == NULL ) {
4136 KMP_FATAL( LockIsUninitialized, func );
4140 if ( OMP_LOCK_T_SIZE <
sizeof(
void *) ) {
4141 kmp_lock_index_t index = *( (kmp_lock_index_t *)user_lock );
4142 if ( __kmp_env_consistency_check ) {
4143 if ( ! ( 0 < index && index < __kmp_user_lock_table.used ) ) {
4144 KMP_FATAL( LockIsUninitialized, func );
4147 KMP_DEBUG_ASSERT( 0 < index && index < __kmp_user_lock_table.used );
4148 KMP_DEBUG_ASSERT( __kmp_user_lock_size > 0 );
4149 lck = __kmp_user_lock_table.table[index];
4152 lck = *( (kmp_user_lock_p *)user_lock );
4155 if ( __kmp_env_consistency_check ) {
4156 if ( lck == NULL ) {
4157 KMP_FATAL( LockIsUninitialized, func );
4165 __kmp_cleanup_user_locks(
void )
4172 __kmp_lock_pool = NULL;
4174 #define IS_CRITICAL(lck) \ 4175 ( ( __kmp_get_user_lock_flags_ != NULL ) && \ 4176 ( ( *__kmp_get_user_lock_flags_ )( lck ) & kmp_lf_critical_section ) ) 4206 while ( __kmp_user_lock_table.used > 1 ) {
4213 kmp_user_lock_p lck = __kmp_user_lock_table.table[
4214 --__kmp_user_lock_table.used ];
4216 if ( ( __kmp_is_user_lock_initialized_ != NULL ) &&
4217 ( *__kmp_is_user_lock_initialized_ )( lck ) ) {
4224 if ( __kmp_env_consistency_check && ( ! IS_CRITICAL( lck ) ) &&
4225 ( ( loc = __kmp_get_user_lock_location( lck ) ) != NULL ) &&
4227 kmp_str_loc_t str_loc = __kmp_str_loc_init( loc->
psource, 0 );
4228 KMP_WARNING( CnsLockNotDestroyed, str_loc.file, str_loc.line );
4229 __kmp_str_loc_free( &str_loc);
4233 if ( IS_CRITICAL( lck ) ) {
4234 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n", lck, *(
void**)lck ) );
4237 KA_TRACE( 20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck, *(
void**)lck ) );
4245 __kmp_destroy_user_lock( lck );
4251 if ( __kmp_lock_blocks == NULL ) {
4261 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4262 __kmp_user_lock_table.table = NULL;
4263 __kmp_user_lock_table.allocated = 0;
4265 while ( table_ptr != NULL ) {
4270 kmp_user_lock_p *next = (kmp_user_lock_p *)( table_ptr[ 0 ] );
4271 __kmp_free( table_ptr );
4278 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4279 __kmp_lock_blocks = NULL;
4281 while ( block_ptr != NULL ) {
4282 kmp_block_of_locks_t *next = block_ptr->next_block;
4283 __kmp_free( block_ptr->locks );
4290 TCW_4(__kmp_init_user_locks, FALSE);
4293 #endif // KMP_USE_DYNAMIC_LOCK