LLVM OpenMP* Runtime Library
kmp_wait_release.h
1 /*
2  * kmp_wait_release.h -- Wait/Release implementation
3  */
4 
5 
6 //===----------------------------------------------------------------------===//
7 //
8 // The LLVM Compiler Infrastructure
9 //
10 // This file is dual licensed under the MIT and the University of Illinois Open
11 // Source Licenses. See LICENSE.txt for details.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 
16 #ifndef KMP_WAIT_RELEASE_H
17 #define KMP_WAIT_RELEASE_H
18 
19 #include "kmp.h"
20 #include "kmp_itt.h"
21 #include "kmp_stats.h"
22 
39 enum flag_type {
43 };
44 
48 template <typename P> class kmp_flag {
49  volatile P
50  *loc;
53 public:
54  typedef P flag_t;
55  kmp_flag(volatile P *p, flag_type ft) : loc(p), t(ft) {}
59  volatile P *get() { return loc; }
63  void set(volatile P *new_loc) { loc = new_loc; }
67  flag_type get_type() { return t; }
68  // Derived classes must provide the following:
69  /*
70  kmp_info_t * get_waiter(kmp_uint32 i);
71  kmp_uint32 get_num_waiters();
72  bool done_check();
73  bool done_check_val(P old_loc);
74  bool notdone_check();
75  P internal_release();
76  void suspend(int th_gtid);
77  void resume(int th_gtid);
78  P set_sleeping();
79  P unset_sleeping();
80  bool is_sleeping();
81  bool is_any_sleeping();
82  bool is_sleeping_val(P old_loc);
83  int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
84  int *thread_finished
85  USE_ITT_BUILD_ARG(void * itt_sync_obj), kmp_int32
86  is_constrained);
87  */
88 };
89 
90 /* Spin wait loop that first does pause, then yield, then sleep. A thread that
91  calls __kmp_wait_* must make certain that another thread calls __kmp_release
92  to wake it back up to prevent deadlocks! */
93 template <class C>
94 static inline void
95 __kmp_wait_template(kmp_info_t *this_thr, C *flag,
96  int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
97  // NOTE: We may not belong to a team at this point.
98  volatile typename C::flag_t *spin = flag->get();
99  kmp_uint32 spins;
100  kmp_uint32 hibernate;
101  int th_gtid;
102  int tasks_completed = FALSE;
103  int oversubscribed;
104 #if !KMP_USE_MONITOR
105  kmp_uint64 poll_count;
106  kmp_uint64 hibernate_goal;
107 #endif
108 
109  KMP_FSYNC_SPIN_INIT(spin, NULL);
110  if (flag->done_check()) {
111  KMP_FSYNC_SPIN_ACQUIRED(CCAST(typename C::flag_t *, spin));
112  return;
113  }
114  th_gtid = this_thr->th.th_info.ds.ds_gtid;
115  KA_TRACE(20,
116  ("__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));
117 #if KMP_STATS_ENABLED
118  stats_state_e thread_state = KMP_GET_THREAD_STATE();
119 #endif
120 
121 #if OMPT_SUPPORT && OMPT_BLAME
122  ompt_state_t ompt_state = this_thr->th.ompt_thread_info.state;
123  if (ompt_enabled && ompt_state != ompt_state_undefined) {
124  if (ompt_state == ompt_state_idle) {
125  if (ompt_callbacks.ompt_callback(ompt_event_idle_begin)) {
126  ompt_callbacks.ompt_callback(ompt_event_idle_begin)(th_gtid + 1);
127  }
128  } else if (ompt_callbacks.ompt_callback(ompt_event_wait_barrier_begin)) {
129  KMP_DEBUG_ASSERT(ompt_state == ompt_state_wait_barrier ||
130  ompt_state == ompt_state_wait_barrier_implicit ||
131  ompt_state == ompt_state_wait_barrier_explicit);
132 
133  ompt_lw_taskteam_t *team =
134  this_thr->th.th_team->t.ompt_serialized_team_info;
135  ompt_parallel_id_t pId;
136  ompt_task_id_t tId;
137  if (team) {
138  pId = team->ompt_team_info.parallel_id;
139  tId = team->ompt_task_info.task_id;
140  } else {
141  pId = this_thr->th.th_team->t.ompt_team_info.parallel_id;
142  tId = this_thr->th.th_current_task->ompt_task_info.task_id;
143  }
144  ompt_callbacks.ompt_callback(ompt_event_wait_barrier_begin)(pId, tId);
145  }
146  }
147 #endif
148 
149  // Setup for waiting
150  KMP_INIT_YIELD(spins);
151 
152  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
153 #if KMP_USE_MONITOR
154 // The worker threads cannot rely on the team struct existing at this point.
155 // Use the bt values cached in the thread struct instead.
156 #ifdef KMP_ADJUST_BLOCKTIME
157  if (__kmp_zero_bt && !this_thr->th.th_team_bt_set)
158  // Force immediate suspend if not set by user and more threads than
159  // available procs
160  hibernate = 0;
161  else
162  hibernate = this_thr->th.th_team_bt_intervals;
163 #else
164  hibernate = this_thr->th.th_team_bt_intervals;
165 #endif /* KMP_ADJUST_BLOCKTIME */
166 
167  /* If the blocktime is nonzero, we want to make sure that we spin wait for
168  the entirety of the specified #intervals, plus up to one interval more.
169  This increment make certain that this thread doesn't go to sleep too
170  soon. */
171  if (hibernate != 0)
172  hibernate++;
173 
174  // Add in the current time value.
175  hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
176  KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
177  th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
178  hibernate - __kmp_global.g.g_time.dt.t_value));
179 #else
180  hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;
181  poll_count = 0;
182 #endif // KMP_USE_MONITOR
183  }
184 
185  oversubscribed = (TCR_4(__kmp_nth) > __kmp_avail_proc);
186  KMP_MB();
187 
188  // Main wait spin loop
189  while (flag->notdone_check()) {
190  int in_pool;
191  kmp_task_team_t *task_team = NULL;
192  if (__kmp_tasking_mode != tskm_immediate_exec) {
193  task_team = this_thr->th.th_task_team;
194  /* If the thread's task team pointer is NULL, it means one of 3 things:
195  1) A newly-created thread is first being released by
196  __kmp_fork_barrier(), and its task team has not been set up yet.
197  2) All tasks have been executed to completion.
198  3) Tasking is off for this region. This could be because we are in a
199  serialized region (perhaps the outer one), or else tasking was manually
200  disabled (KMP_TASKING=0). */
201  if (task_team != NULL) {
202  if (TCR_SYNC_4(task_team->tt.tt_active)) {
203  if (KMP_TASKING_ENABLED(task_team))
204  flag->execute_tasks(
205  this_thr, th_gtid, final_spin,
206  &tasks_completed USE_ITT_BUILD_ARG(itt_sync_obj), 0);
207  else
208  this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
209  } else {
210  KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));
211  this_thr->th.th_task_team = NULL;
212  this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
213  }
214  } else {
215  this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
216  } // if
217  } // if
218 
219  KMP_FSYNC_SPIN_PREPARE(CCAST(typename C::flag_t *, spin));
220  if (TCR_4(__kmp_global.g.g_done)) {
221  if (__kmp_global.g.g_abort)
222  __kmp_abort_thread();
223  break;
224  }
225 
226  // If we are oversubscribed, or have waited a bit (and
227  // KMP_LIBRARY=throughput), then yield
228  // TODO: Should it be number of cores instead of thread contexts? Like:
229  // KMP_YIELD(TCR_4(__kmp_nth) > __kmp_ncores);
230  // Need performance improvement data to make the change...
231  if (oversubscribed) {
232  KMP_YIELD(1);
233  } else {
234  KMP_YIELD_SPIN(spins);
235  }
236  // Check if this thread was transferred from a team
237  // to the thread pool (or vice-versa) while spinning.
238  in_pool = !!TCR_4(this_thr->th.th_in_pool);
239  if (in_pool != !!this_thr->th.th_active_in_pool) {
240  if (in_pool) { // Recently transferred from team to pool
241  KMP_TEST_THEN_INC32(&__kmp_thread_pool_active_nth);
242  this_thr->th.th_active_in_pool = TRUE;
243  /* Here, we cannot assert that:
244  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) <=
245  __kmp_thread_pool_nth);
246  __kmp_thread_pool_nth is inc/dec'd by the master thread while the
247  fork/join lock is held, whereas __kmp_thread_pool_active_nth is
248  inc/dec'd asynchronously by the workers. The two can get out of sync
249  for brief periods of time. */
250  } else { // Recently transferred from pool to team
251  KMP_TEST_THEN_DEC32(&__kmp_thread_pool_active_nth);
252  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
253  this_thr->th.th_active_in_pool = FALSE;
254  }
255  }
256 
257 #if KMP_STATS_ENABLED
258  // Check if thread has been signalled to idle state
259  // This indicates that the logical "join-barrier" has finished
260  if (this_thr->th.th_stats->isIdle() &&
261  KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {
262  KMP_SET_THREAD_STATE(IDLE);
263  KMP_PUSH_PARTITIONED_TIMER(OMP_idle);
264  }
265 #endif
266 
267  // Don't suspend if KMP_BLOCKTIME is set to "infinite"
268  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
269  continue;
270 
271  // Don't suspend if there is a likelihood of new tasks being spawned.
272  if ((task_team != NULL) && TCR_4(task_team->tt.tt_found_tasks))
273  continue;
274 
275 #if KMP_USE_MONITOR
276  // If we have waited a bit more, fall asleep
277  if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
278  continue;
279 #else
280  if (KMP_BLOCKING(hibernate_goal, poll_count++))
281  continue;
282 #endif
283 
284  KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
285  flag->suspend(th_gtid);
286 
287  if (TCR_4(__kmp_global.g.g_done)) {
288  if (__kmp_global.g.g_abort)
289  __kmp_abort_thread();
290  break;
291  } else if (__kmp_tasking_mode != tskm_immediate_exec &&
292  this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) {
293  this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
294  }
295  // TODO: If thread is done with work and times out, disband/free
296  }
297 
298 #if OMPT_SUPPORT && OMPT_BLAME
299  if (ompt_enabled && ompt_state != ompt_state_undefined) {
300  if (ompt_state == ompt_state_idle) {
301  if (ompt_callbacks.ompt_callback(ompt_event_idle_end)) {
302  ompt_callbacks.ompt_callback(ompt_event_idle_end)(th_gtid + 1);
303  }
304  } else if (ompt_callbacks.ompt_callback(ompt_event_wait_barrier_end)) {
305  KMP_DEBUG_ASSERT(ompt_state == ompt_state_wait_barrier ||
306  ompt_state == ompt_state_wait_barrier_implicit ||
307  ompt_state == ompt_state_wait_barrier_explicit);
308 
309  ompt_lw_taskteam_t *team =
310  this_thr->th.th_team->t.ompt_serialized_team_info;
311  ompt_parallel_id_t pId;
312  ompt_task_id_t tId;
313  if (team) {
314  pId = team->ompt_team_info.parallel_id;
315  tId = team->ompt_task_info.task_id;
316  } else {
317  pId = this_thr->th.th_team->t.ompt_team_info.parallel_id;
318  tId = this_thr->th.th_current_task->ompt_task_info.task_id;
319  }
320  ompt_callbacks.ompt_callback(ompt_event_wait_barrier_end)(pId, tId);
321  }
322  }
323 #endif
324 #if KMP_STATS_ENABLED
325  // If we were put into idle state, pop that off the state stack
326  if (KMP_GET_THREAD_STATE() == IDLE) {
327  KMP_POP_PARTITIONED_TIMER();
328  KMP_SET_THREAD_STATE(thread_state);
329  this_thr->th.th_stats->resetIdleFlag();
330  }
331 #endif
332 
333  KMP_FSYNC_SPIN_ACQUIRED(CCAST(typename C::flag_t *, spin));
334 }
335 
336 /* Release any threads specified as waiting on the flag by releasing the flag
337  and resume the waiting thread if indicated by the sleep bit(s). A thread that
338  calls __kmp_wait_template must call this function to wake up the potentially
339  sleeping thread and prevent deadlocks! */
340 template <class C> static inline void __kmp_release_template(C *flag) {
341 #ifdef KMP_DEBUG
342  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
343 #endif
344  KF_TRACE(20, ("__kmp_release: T#%d releasing flag(%x)\n", gtid, flag->get()));
345  KMP_DEBUG_ASSERT(flag->get());
346  KMP_FSYNC_RELEASING(CCAST(typename C::flag_t *, flag->get()));
347 
348  flag->internal_release();
349 
350  KF_TRACE(100, ("__kmp_release: T#%d set new spin=%d\n", gtid, flag->get(),
351  *(flag->get())));
352 
353  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
354  // Only need to check sleep stuff if infinite block time not set.
355  // Are *any* threads waiting on flag sleeping?
356  if (flag->is_any_sleeping()) {
357  for (unsigned int i = 0; i < flag->get_num_waiters(); ++i) {
358  // if sleeping waiter exists at i, sets current_waiter to i inside flag
359  kmp_info_t *waiter = flag->get_waiter(i);
360  if (waiter) {
361  int wait_gtid = waiter->th.th_info.ds.ds_gtid;
362  // Wake up thread if needed
363  KF_TRACE(50, ("__kmp_release: T#%d waking up thread T#%d since sleep "
364  "flag(%p) set\n",
365  gtid, wait_gtid, flag->get()));
366  flag->resume(wait_gtid); // unsets flag's current_waiter when done
367  }
368  }
369  }
370  }
371 }
372 
373 template <typename FlagType> struct flag_traits {};
374 
375 template <> struct flag_traits<kmp_uint32> {
376  typedef kmp_uint32 flag_t;
377  static const flag_type t = flag32;
378  static inline flag_t tcr(flag_t f) { return TCR_4(f); }
379  static inline flag_t test_then_add4(volatile flag_t *f) {
380  return KMP_TEST_THEN_ADD4_32(RCAST(volatile kmp_int32 *, f));
381  }
382  static inline flag_t test_then_or(volatile flag_t *f, flag_t v) {
383  return KMP_TEST_THEN_OR32(f, v);
384  }
385  static inline flag_t test_then_and(volatile flag_t *f, flag_t v) {
386  return KMP_TEST_THEN_AND32(f, v);
387  }
388 };
389 
390 template <> struct flag_traits<kmp_uint64> {
391  typedef kmp_uint64 flag_t;
392  static const flag_type t = flag64;
393  static inline flag_t tcr(flag_t f) { return TCR_8(f); }
394  static inline flag_t test_then_add4(volatile flag_t *f) {
395  return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f));
396  }
397  static inline flag_t test_then_or(volatile flag_t *f, flag_t v) {
398  return KMP_TEST_THEN_OR64(f, v);
399  }
400  static inline flag_t test_then_and(volatile flag_t *f, flag_t v) {
401  return KMP_TEST_THEN_AND64(f, v);
402  }
403 };
404 
405 template <typename FlagType> class kmp_basic_flag : public kmp_flag<FlagType> {
406  typedef flag_traits<FlagType> traits_type;
407  FlagType checker;
409  kmp_info_t
410  *waiting_threads[1];
411  kmp_uint32
412  num_waiting_threads;
413 public:
414  kmp_basic_flag(volatile FlagType *p)
415  : kmp_flag<FlagType>(p, traits_type::t), num_waiting_threads(0) {}
416  kmp_basic_flag(volatile FlagType *p, kmp_info_t *thr)
417  : kmp_flag<FlagType>(p, traits_type::t), num_waiting_threads(1) {
418  waiting_threads[0] = thr;
419  }
420  kmp_basic_flag(volatile FlagType *p, FlagType c)
421  : kmp_flag<FlagType>(p, traits_type::t), checker(c),
422  num_waiting_threads(0) {}
427  kmp_info_t *get_waiter(kmp_uint32 i) {
428  KMP_DEBUG_ASSERT(i < num_waiting_threads);
429  return waiting_threads[i];
430  }
434  kmp_uint32 get_num_waiters() { return num_waiting_threads; }
440  void set_waiter(kmp_info_t *thr) {
441  waiting_threads[0] = thr;
442  num_waiting_threads = 1;
443  }
447  bool done_check() { return traits_type::tcr(*(this->get())) == checker; }
452  bool done_check_val(FlagType old_loc) { return old_loc == checker; }
460  bool notdone_check() { return traits_type::tcr(*(this->get())) != checker; }
465  void internal_release() {
466  (void)traits_type::test_then_add4((volatile FlagType *)this->get());
467  }
473  FlagType set_sleeping() {
474  return traits_type::test_then_or((volatile FlagType *)this->get(),
475  KMP_BARRIER_SLEEP_STATE);
476  }
482  FlagType unset_sleeping() {
483  return traits_type::test_then_and((volatile FlagType *)this->get(),
484  ~KMP_BARRIER_SLEEP_STATE);
485  }
490  bool is_sleeping_val(FlagType old_loc) {
491  return old_loc & KMP_BARRIER_SLEEP_STATE;
492  }
496  bool is_sleeping() { return is_sleeping_val(*(this->get())); }
497  bool is_any_sleeping() { return is_sleeping_val(*(this->get())); }
498  kmp_uint8 *get_stolen() { return NULL; }
499  enum barrier_type get_bt() { return bs_last_barrier; }
500 };
501 
502 class kmp_flag_32 : public kmp_basic_flag<kmp_uint32> {
503 public:
504  kmp_flag_32(volatile kmp_uint32 *p) : kmp_basic_flag<kmp_uint32>(p) {}
505  kmp_flag_32(volatile kmp_uint32 *p, kmp_info_t *thr)
506  : kmp_basic_flag<kmp_uint32>(p, thr) {}
507  kmp_flag_32(volatile kmp_uint32 *p, kmp_uint32 c)
508  : kmp_basic_flag<kmp_uint32>(p, c) {}
509  void suspend(int th_gtid) { __kmp_suspend_32(th_gtid, this); }
510  void resume(int th_gtid) { __kmp_resume_32(th_gtid, this); }
511  int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
512  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
513  kmp_int32 is_constrained) {
514  return __kmp_execute_tasks_32(
515  this_thr, gtid, this, final_spin,
516  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
517  }
518  void wait(kmp_info_t *this_thr,
519  int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
520  __kmp_wait_template(this_thr, this,
521  final_spin USE_ITT_BUILD_ARG(itt_sync_obj));
522  }
523  void release() { __kmp_release_template(this); }
524  flag_type get_ptr_type() { return flag32; }
525 };
526 
527 class kmp_flag_64 : public kmp_basic_flag<kmp_uint64> {
528 public:
529  kmp_flag_64(volatile kmp_uint64 *p) : kmp_basic_flag<kmp_uint64>(p) {}
530  kmp_flag_64(volatile kmp_uint64 *p, kmp_info_t *thr)
531  : kmp_basic_flag<kmp_uint64>(p, thr) {}
532  kmp_flag_64(volatile kmp_uint64 *p, kmp_uint64 c)
533  : kmp_basic_flag<kmp_uint64>(p, c) {}
534  void suspend(int th_gtid) { __kmp_suspend_64(th_gtid, this); }
535  void resume(int th_gtid) { __kmp_resume_64(th_gtid, this); }
536  int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
537  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
538  kmp_int32 is_constrained) {
539  return __kmp_execute_tasks_64(
540  this_thr, gtid, this, final_spin,
541  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
542  }
543  void wait(kmp_info_t *this_thr,
544  int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
545  __kmp_wait_template(this_thr, this,
546  final_spin USE_ITT_BUILD_ARG(itt_sync_obj));
547  }
548  void release() { __kmp_release_template(this); }
549  flag_type get_ptr_type() { return flag64; }
550 };
551 
552 // Hierarchical 64-bit on-core barrier instantiation
553 class kmp_flag_oncore : public kmp_flag<kmp_uint64> {
554  kmp_uint64 checker;
555  kmp_info_t *waiting_threads[1];
556  kmp_uint32 num_waiting_threads;
557  kmp_uint32
558  offset;
559  bool flag_switch;
560  enum barrier_type bt;
561  kmp_info_t *this_thr;
563 #if USE_ITT_BUILD
564  void *
565  itt_sync_obj;
566 #endif
567  unsigned char &byteref(volatile kmp_uint64 *loc, size_t offset) {
568  return (RCAST(unsigned char *, CCAST(kmp_uint64 *, loc)))[offset];
569  }
570 
571 public:
572  kmp_flag_oncore(volatile kmp_uint64 *p)
573  : kmp_flag<kmp_uint64>(p, flag_oncore), num_waiting_threads(0),
574  flag_switch(false) {}
575  kmp_flag_oncore(volatile kmp_uint64 *p, kmp_uint32 idx)
576  : kmp_flag<kmp_uint64>(p, flag_oncore), num_waiting_threads(0),
577  offset(idx), flag_switch(false) {}
578  kmp_flag_oncore(volatile kmp_uint64 *p, kmp_uint64 c, kmp_uint32 idx,
579  enum barrier_type bar_t, kmp_info_t *thr
580 #if USE_ITT_BUILD
581  ,
582  void *itt
583 #endif
584  )
585  : kmp_flag<kmp_uint64>(p, flag_oncore), checker(c),
586  num_waiting_threads(0), offset(idx), flag_switch(false), bt(bar_t),
587  this_thr(thr)
588 #if USE_ITT_BUILD
589  ,
590  itt_sync_obj(itt)
591 #endif
592  {
593  }
594  kmp_info_t *get_waiter(kmp_uint32 i) {
595  KMP_DEBUG_ASSERT(i < num_waiting_threads);
596  return waiting_threads[i];
597  }
598  kmp_uint32 get_num_waiters() { return num_waiting_threads; }
599  void set_waiter(kmp_info_t *thr) {
600  waiting_threads[0] = thr;
601  num_waiting_threads = 1;
602  }
603  bool done_check_val(kmp_uint64 old_loc) {
604  return byteref(&old_loc, offset) == checker;
605  }
606  bool done_check() { return done_check_val(*get()); }
607  bool notdone_check() {
608  // Calculate flag_switch
609  if (this_thr->th.th_bar[bt].bb.wait_flag == KMP_BARRIER_SWITCH_TO_OWN_FLAG)
610  flag_switch = true;
611  if (byteref(get(), offset) != 1 && !flag_switch)
612  return true;
613  else if (flag_switch) {
614  this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING;
615  kmp_flag_64 flag(&this_thr->th.th_bar[bt].bb.b_go,
616  (kmp_uint64)KMP_BARRIER_STATE_BUMP);
617  __kmp_wait_64(this_thr, &flag, TRUE
618 #if USE_ITT_BUILD
619  ,
620  itt_sync_obj
621 #endif
622  );
623  }
624  return false;
625  }
626  void internal_release() {
627  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
628  byteref(get(), offset) = 1;
629  } else {
630  kmp_uint64 mask = 0;
631  byteref(&mask, offset) = 1;
632  KMP_TEST_THEN_OR64(get(), mask);
633  }
634  }
635  kmp_uint64 set_sleeping() {
636  return KMP_TEST_THEN_OR64(get(), KMP_BARRIER_SLEEP_STATE);
637  }
638  kmp_uint64 unset_sleeping() {
639  return KMP_TEST_THEN_AND64(get(), ~KMP_BARRIER_SLEEP_STATE);
640  }
641  bool is_sleeping_val(kmp_uint64 old_loc) {
642  return old_loc & KMP_BARRIER_SLEEP_STATE;
643  }
644  bool is_sleeping() { return is_sleeping_val(*get()); }
645  bool is_any_sleeping() { return is_sleeping_val(*get()); }
646  void wait(kmp_info_t *this_thr, int final_spin) {
647  __kmp_wait_template<kmp_flag_oncore>(
648  this_thr, this, final_spin USE_ITT_BUILD_ARG(itt_sync_obj));
649  }
650  void release() { __kmp_release_template(this); }
651  void suspend(int th_gtid) { __kmp_suspend_oncore(th_gtid, this); }
652  void resume(int th_gtid) { __kmp_resume_oncore(th_gtid, this); }
653  int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
654  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
655  kmp_int32 is_constrained) {
656  return __kmp_execute_tasks_oncore(
657  this_thr, gtid, this, final_spin,
658  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
659  }
660  kmp_uint8 *get_stolen() { return NULL; }
661  enum barrier_type get_bt() { return bt; }
662  flag_type get_ptr_type() { return flag_oncore; }
663 };
664 
665 // Used to wake up threads, volatile void* flag is usually the th_sleep_loc
666 // associated with int gtid.
667 static inline void __kmp_null_resume_wrapper(int gtid, volatile void *flag) {
668  if (!flag)
669  return;
670 
671  switch (RCAST(kmp_flag_64 *, CCAST(void *, flag))->get_type()) {
672  case flag32:
673  __kmp_resume_32(gtid, NULL);
674  break;
675  case flag64:
676  __kmp_resume_64(gtid, NULL);
677  break;
678  case flag_oncore:
679  __kmp_resume_oncore(gtid, NULL);
680  break;
681  }
682 }
683 
688 #endif // KMP_WAIT_RELEASE_H
volatile P * loc
flag_type get_type()
flag_type
flag_type t
stats_state_e
the states which a thread can be in
Definition: kmp_stats.h:63