15 #include "kmp_affinity.h" 19 #include "kmp_wrapper_getpid.h" 22 static hierarchy_info machine_hierarchy;
24 void __kmp_cleanup_hierarchy() { machine_hierarchy.fini(); }
26 void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar) {
30 if (TCR_1(machine_hierarchy.uninitialized))
31 machine_hierarchy.init(NULL, nproc);
34 if (nproc > machine_hierarchy.base_num_threads)
35 machine_hierarchy.resize(nproc);
37 depth = machine_hierarchy.depth;
38 KMP_DEBUG_ASSERT(depth > 0);
40 thr_bar->depth = depth;
41 thr_bar->base_leaf_kids = (kmp_uint8)machine_hierarchy.numPerLevel[0] - 1;
42 thr_bar->skip_per_level = machine_hierarchy.skipPerLevel;
45 #if KMP_AFFINITY_SUPPORTED 47 bool KMPAffinity::picked_api =
false;
49 void *KMPAffinity::Mask::operator
new(
size_t n) {
return __kmp_allocate(n); }
50 void *KMPAffinity::Mask::operator
new[](
size_t n) {
return __kmp_allocate(n); }
51 void KMPAffinity::Mask::operator
delete(
void *p) { __kmp_free(p); }
52 void KMPAffinity::Mask::operator
delete[](
void *p) { __kmp_free(p); }
53 void *KMPAffinity::operator
new(
size_t n) {
return __kmp_allocate(n); }
54 void KMPAffinity::operator
delete(
void *p) { __kmp_free(p); }
56 void KMPAffinity::pick_api() {
57 KMPAffinity *affinity_dispatch;
63 if (__kmp_affinity_top_method == affinity_top_method_hwloc &&
64 __kmp_affinity_type != affinity_disabled) {
65 affinity_dispatch =
new KMPHwlocAffinity();
69 affinity_dispatch =
new KMPNativeAffinity();
71 __kmp_affinity_dispatch = affinity_dispatch;
75 void KMPAffinity::destroy_api() {
76 if (__kmp_affinity_dispatch != NULL) {
77 delete __kmp_affinity_dispatch;
78 __kmp_affinity_dispatch = NULL;
84 char *__kmp_affinity_print_mask(
char *buf,
int buf_len,
85 kmp_affin_mask_t *mask) {
86 KMP_ASSERT(buf_len >= 40);
88 char *end = buf + buf_len - 1;
93 if (i == mask->end()) {
94 KMP_SNPRINTF(scan, end - scan + 1,
"{<empty>}");
97 KMP_ASSERT(scan <= end);
101 KMP_SNPRINTF(scan, end - scan + 1,
"{%ld", (
long)i);
102 while (*scan !=
'\0')
105 for (; i != mask->end(); i = mask->next(i)) {
106 if (!KMP_CPU_ISSET(i, mask)) {
114 if (end - scan < 15) {
117 KMP_SNPRINTF(scan, end - scan + 1,
",%-ld", (
long)i);
118 while (*scan !=
'\0')
121 if (i != mask->end()) {
122 KMP_SNPRINTF(scan, end - scan + 1,
",...");
123 while (*scan !=
'\0')
126 KMP_SNPRINTF(scan, end - scan + 1,
"}");
127 while (*scan !=
'\0')
129 KMP_ASSERT(scan <= end);
133 void __kmp_affinity_entire_machine_mask(kmp_affin_mask_t *mask) {
136 #if KMP_GROUP_AFFINITY 138 if (__kmp_num_proc_groups > 1) {
140 KMP_DEBUG_ASSERT(__kmp_GetActiveProcessorCount != NULL);
141 for (group = 0; group < __kmp_num_proc_groups; group++) {
143 int num = __kmp_GetActiveProcessorCount(group);
144 for (i = 0; i < num; i++) {
145 KMP_CPU_SET(i + group * (CHAR_BIT *
sizeof(DWORD_PTR)), mask);
154 for (proc = 0; proc < __kmp_xproc; proc++) {
155 KMP_CPU_SET(proc, mask);
171 static void __kmp_affinity_assign_child_nums(AddrUnsPair *address2os,
173 KMP_DEBUG_ASSERT(numAddrs > 0);
174 int depth = address2os->first.depth;
175 unsigned *counts = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
176 unsigned *lastLabel = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
178 for (labCt = 0; labCt < depth; labCt++) {
179 address2os[0].first.childNums[labCt] = counts[labCt] = 0;
180 lastLabel[labCt] = address2os[0].first.labels[labCt];
183 for (i = 1; i < numAddrs; i++) {
184 for (labCt = 0; labCt < depth; labCt++) {
185 if (address2os[i].first.labels[labCt] != lastLabel[labCt]) {
187 for (labCt2 = labCt + 1; labCt2 < depth; labCt2++) {
189 lastLabel[labCt2] = address2os[i].first.labels[labCt2];
192 lastLabel[labCt] = address2os[i].first.labels[labCt];
196 for (labCt = 0; labCt < depth; labCt++) {
197 address2os[i].first.childNums[labCt] = counts[labCt];
199 for (; labCt < (int)Address::maxDepth; labCt++) {
200 address2os[i].first.childNums[labCt] = 0;
203 __kmp_free(lastLabel);
218 kmp_affin_mask_t *__kmp_affin_fullMask = NULL;
220 static int nCoresPerPkg, nPackages;
221 static int __kmp_nThreadsPerCore;
222 #ifndef KMP_DFLT_NTH_CORES 223 static int __kmp_ncores;
225 static int *__kmp_pu_os_idx = NULL;
231 inline static bool __kmp_affinity_uniform_topology() {
232 return __kmp_avail_proc == (__kmp_nThreadsPerCore * nCoresPerPkg * nPackages);
237 static void __kmp_affinity_print_topology(AddrUnsPair *address2os,
int len,
238 int depth,
int pkgLevel,
239 int coreLevel,
int threadLevel) {
242 KMP_INFORM(OSProcToPhysicalThreadMap,
"KMP_AFFINITY");
243 for (proc = 0; proc < len; proc++) {
246 __kmp_str_buf_init(&buf);
247 for (level = 0; level < depth; level++) {
248 if (level == threadLevel) {
249 __kmp_str_buf_print(&buf,
"%s ", KMP_I18N_STR(Thread));
250 }
else if (level == coreLevel) {
251 __kmp_str_buf_print(&buf,
"%s ", KMP_I18N_STR(Core));
252 }
else if (level == pkgLevel) {
253 __kmp_str_buf_print(&buf,
"%s ", KMP_I18N_STR(Package));
254 }
else if (level > pkgLevel) {
255 __kmp_str_buf_print(&buf,
"%s_%d ", KMP_I18N_STR(Node),
256 level - pkgLevel - 1);
258 __kmp_str_buf_print(&buf,
"L%d ", level);
260 __kmp_str_buf_print(&buf,
"%d ", address2os[proc].first.labels[level]);
262 KMP_INFORM(OSProcMapToPack,
"KMP_AFFINITY", address2os[proc].second,
264 __kmp_str_buf_free(&buf);
270 static void __kmp_affinity_print_hwloc_tp(AddrUnsPair *addrP,
int len,
271 int depth,
int *levels) {
274 __kmp_str_buf_init(&buf);
275 KMP_INFORM(OSProcToPhysicalThreadMap,
"KMP_AFFINITY");
276 for (proc = 0; proc < len; proc++) {
277 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Package),
278 addrP[proc].first.labels[0]);
282 if (__kmp_numa_detected)
284 if (levels[level++] > 0)
285 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Node),
286 addrP[proc].first.labels[label++]);
287 if (__kmp_tile_depth > 0)
289 if (levels[level++] > 0)
290 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Tile),
291 addrP[proc].first.labels[label++]);
292 if (levels[level++] > 0)
294 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Core),
295 addrP[proc].first.labels[label++]);
296 if (levels[level++] > 0)
298 __kmp_str_buf_print(&buf,
"%s %d ", KMP_I18N_STR(Thread),
299 addrP[proc].first.labels[label++]);
300 KMP_DEBUG_ASSERT(label == depth);
302 KMP_INFORM(OSProcMapToPack,
"KMP_AFFINITY", addrP[proc].second, buf.str);
303 __kmp_str_buf_clear(&buf);
305 __kmp_str_buf_free(&buf);
308 static int nNodePerPkg, nTilePerPkg, nTilePerNode, nCorePerNode, nCorePerTile;
315 static int __kmp_affinity_remove_radix_one_levels(AddrUnsPair *addrP,
int nTh,
316 int depth,
int *levels) {
320 int new_depth = depth;
321 for (level = depth - 1; level > 0; --level) {
324 for (i = 1; i < nTh; ++i) {
325 if (addrP[0].first.labels[level] != addrP[i].first.labels[level]) {
331 if (!radix1_detected)
336 if (level == new_depth) {
339 for (i = 0; i < nTh; ++i) {
340 addrP[i].first.depth--;
345 for (j = level; j < new_depth; ++j) {
346 for (i = 0; i < nTh; ++i) {
347 addrP[i].first.labels[j] = addrP[i].first.labels[j + 1];
348 addrP[i].first.depth--;
361 static int __kmp_hwloc_get_nobjs_under_obj(hwloc_obj_t obj,
362 hwloc_obj_type_t type) {
365 for (first = hwloc_get_obj_below_by_type(__kmp_hwloc_topology, obj->type,
366 obj->logical_index, type, 0);
368 hwloc_get_ancestor_obj_by_type(__kmp_hwloc_topology, obj->type, first) ==
370 first = hwloc_get_next_obj_by_type(__kmp_hwloc_topology, first->type,
377 static int __kmp_hwloc_count_children_by_depth(hwloc_topology_t t,
378 hwloc_obj_t o,
unsigned depth,
380 if (o->depth == depth) {
386 for (
unsigned i = 0; i < o->arity; i++)
387 sum += __kmp_hwloc_count_children_by_depth(t, o->children[i], depth, f);
391 static int __kmp_hwloc_count_children_by_type(hwloc_topology_t t, hwloc_obj_t o,
392 hwloc_obj_type_t type,
394 if (!hwloc_compare_types(o->type, type)) {
400 for (
unsigned i = 0; i < o->arity; i++)
401 sum += __kmp_hwloc_count_children_by_type(t, o->children[i], type, f);
405 static int __kmp_hwloc_process_obj_core_pu(AddrUnsPair *addrPair,
407 int &num_active_cores,
408 hwloc_obj_t obj,
int depth,
410 hwloc_obj_t core = NULL;
411 hwloc_topology_t &tp = __kmp_hwloc_topology;
412 int NC = __kmp_hwloc_count_children_by_type(tp, obj, HWLOC_OBJ_CORE, &core);
413 for (
int core_id = 0; core_id < NC; ++core_id, core = core->next_cousin) {
414 hwloc_obj_t pu = NULL;
415 KMP_DEBUG_ASSERT(core != NULL);
416 int num_active_threads = 0;
417 int NT = __kmp_hwloc_count_children_by_type(tp, core, HWLOC_OBJ_PU, &pu);
419 for (
int pu_id = 0; pu_id < NT; ++pu_id, pu = pu->next_cousin) {
420 KMP_DEBUG_ASSERT(pu != NULL);
421 if (!KMP_CPU_ISSET(pu->os_index, __kmp_affin_fullMask))
423 Address addr(depth + 2);
424 KA_TRACE(20, (
"Hwloc inserting %d (%d) %d (%d) %d (%d) into address2os\n",
425 obj->os_index, obj->logical_index, core->os_index,
426 core->logical_index, pu->os_index, pu->logical_index));
427 for (
int i = 0; i < depth; ++i)
428 addr.labels[i] = labels[i];
429 addr.labels[depth] = core_id;
430 addr.labels[depth + 1] = pu_id;
431 addrPair[nActiveThreads] = AddrUnsPair(addr, pu->os_index);
432 __kmp_pu_os_idx[nActiveThreads] = pu->os_index;
434 ++num_active_threads;
436 if (num_active_threads) {
439 if (num_active_threads > __kmp_nThreadsPerCore)
440 __kmp_nThreadsPerCore = num_active_threads;
448 static int __kmp_hwloc_check_numa() {
449 hwloc_topology_t &tp = __kmp_hwloc_topology;
450 hwloc_obj_t hT, hC, hL, hN, hS;
454 hT = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PU, 0);
459 hN = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hT);
460 hS = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hT);
461 KMP_DEBUG_ASSERT(hS != NULL);
462 if (hN != NULL && hN->depth > hS->depth) {
463 __kmp_numa_detected = TRUE;
464 if (__kmp_affinity_gran == affinity_gran_node) {
465 __kmp_affinity_gran == affinity_gran_numa;
470 depth = hwloc_get_cache_type_depth(tp, 2, HWLOC_OBJ_CACHE_UNIFIED);
471 hL = hwloc_get_ancestor_obj_by_depth(tp, depth, hT);
474 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC) > 1)
475 __kmp_tile_depth = depth;
479 static int __kmp_affinity_create_hwloc_map(AddrUnsPair **address2os,
480 kmp_i18n_id_t *
const msg_id) {
481 hwloc_topology_t &tp = __kmp_hwloc_topology;
483 *msg_id = kmp_i18n_null;
486 kmp_affin_mask_t *oldMask;
487 KMP_CPU_ALLOC(oldMask);
488 __kmp_get_system_affinity(oldMask, TRUE);
489 __kmp_hwloc_check_numa();
491 if (!KMP_AFFINITY_CAPABLE()) {
494 KMP_ASSERT(__kmp_affinity_type == affinity_none);
496 nCoresPerPkg = __kmp_hwloc_get_nobjs_under_obj(
497 hwloc_get_obj_by_type(tp, HWLOC_OBJ_PACKAGE, 0), HWLOC_OBJ_CORE);
498 __kmp_nThreadsPerCore = __kmp_hwloc_get_nobjs_under_obj(
499 hwloc_get_obj_by_type(tp, HWLOC_OBJ_CORE, 0), HWLOC_OBJ_PU);
500 __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore;
501 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
502 if (__kmp_affinity_verbose) {
503 KMP_INFORM(AffNotCapableUseLocCpuidL11,
"KMP_AFFINITY");
504 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
505 if (__kmp_affinity_uniform_topology()) {
506 KMP_INFORM(Uniform,
"KMP_AFFINITY");
508 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
510 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
511 __kmp_nThreadsPerCore, __kmp_ncores);
513 KMP_CPU_FREE(oldMask);
518 int levels[5] = {0, 1, 2, 3, 4};
520 if (__kmp_numa_detected)
522 if (__kmp_tile_depth)
526 AddrUnsPair *retval =
527 (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * __kmp_avail_proc);
528 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
529 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
536 hwloc_obj_t socket, node, tile;
537 int nActiveThreads = 0;
540 __kmp_ncores = nPackages = nCoresPerPkg = __kmp_nThreadsPerCore = 0;
541 nNodePerPkg = nTilePerPkg = nTilePerNode = nCorePerNode = nCorePerTile = 0;
542 for (socket = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PACKAGE, 0); socket != NULL;
543 socket = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PACKAGE, socket),
545 labels[0] = socket_id;
546 if (__kmp_numa_detected) {
548 int n_active_nodes = 0;
550 NN = __kmp_hwloc_count_children_by_type(tp, socket, HWLOC_OBJ_NUMANODE,
552 for (
int node_id = 0; node_id < NN; ++node_id, node = node->next_cousin) {
554 if (__kmp_tile_depth) {
557 int n_active_tiles = 0;
559 NT = __kmp_hwloc_count_children_by_depth(tp, node, __kmp_tile_depth,
561 for (
int tl_id = 0; tl_id < NT; ++tl_id, tile = tile->next_cousin) {
563 int n_active_cores = 0;
564 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
565 n_active_cores, tile, 3, labels);
566 if (n_active_cores) {
568 if (n_active_cores > nCorePerTile)
569 nCorePerTile = n_active_cores;
572 if (n_active_tiles) {
574 if (n_active_tiles > nTilePerNode)
575 nTilePerNode = n_active_tiles;
579 int n_active_cores = 0;
580 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
581 n_active_cores, node, 2, labels);
582 if (n_active_cores) {
584 if (n_active_cores > nCorePerNode)
585 nCorePerNode = n_active_cores;
589 if (n_active_nodes) {
591 if (n_active_nodes > nNodePerPkg)
592 nNodePerPkg = n_active_nodes;
595 if (__kmp_tile_depth) {
598 int n_active_tiles = 0;
600 NT = __kmp_hwloc_count_children_by_depth(tp, socket, __kmp_tile_depth,
602 for (
int tl_id = 0; tl_id < NT; ++tl_id, tile = tile->next_cousin) {
604 int n_active_cores = 0;
605 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads,
606 n_active_cores, tile, 2, labels);
607 if (n_active_cores) {
609 if (n_active_cores > nCorePerTile)
610 nCorePerTile = n_active_cores;
613 if (n_active_tiles) {
615 if (n_active_tiles > nTilePerPkg)
616 nTilePerPkg = n_active_tiles;
620 int n_active_cores = 0;
621 __kmp_hwloc_process_obj_core_pu(retval, nActiveThreads, n_active_cores,
623 if (n_active_cores) {
625 if (n_active_cores > nCoresPerPkg)
626 nCoresPerPkg = n_active_cores;
633 KMP_DEBUG_ASSERT(nActiveThreads == __kmp_avail_proc);
634 KMP_ASSERT(nActiveThreads > 0);
635 if (nActiveThreads == 1) {
636 __kmp_ncores = nPackages = 1;
637 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
638 if (__kmp_affinity_verbose) {
639 char buf[KMP_AFFIN_MASK_PRINT_LEN];
640 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
642 KMP_INFORM(AffUsingHwloc,
"KMP_AFFINITY");
643 if (__kmp_affinity_respect_mask) {
644 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
646 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
648 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
649 KMP_INFORM(Uniform,
"KMP_AFFINITY");
650 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
651 __kmp_nThreadsPerCore, __kmp_ncores);
654 if (__kmp_affinity_type == affinity_none) {
656 KMP_CPU_FREE(oldMask);
662 addr.labels[0] = retval[0].first.labels[0];
663 retval[0].first = addr;
665 if (__kmp_affinity_gran_levels < 0) {
666 __kmp_affinity_gran_levels = 0;
669 if (__kmp_affinity_verbose) {
670 __kmp_affinity_print_topology(retval, 1, 1, 0, -1, -1);
673 *address2os = retval;
674 KMP_CPU_FREE(oldMask);
679 qsort(retval, nActiveThreads,
sizeof(*retval),
680 __kmp_affinity_cmp_Address_labels);
683 int nPUs = nPackages * __kmp_nThreadsPerCore;
684 if (__kmp_numa_detected) {
685 if (__kmp_tile_depth) {
686 nPUs *= (nNodePerPkg * nTilePerNode * nCorePerTile);
688 nPUs *= (nNodePerPkg * nCorePerNode);
691 if (__kmp_tile_depth) {
692 nPUs *= (nTilePerPkg * nCorePerTile);
694 nPUs *= nCoresPerPkg;
697 unsigned uniform = (nPUs == nActiveThreads);
700 if (__kmp_affinity_verbose) {
701 char mask[KMP_AFFIN_MASK_PRINT_LEN];
702 __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
703 if (__kmp_affinity_respect_mask) {
704 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", mask);
706 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", mask);
708 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
710 KMP_INFORM(Uniform,
"KMP_AFFINITY");
712 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
714 if (__kmp_numa_detected) {
715 if (__kmp_tile_depth) {
716 KMP_INFORM(TopologyExtraNoTi,
"KMP_AFFINITY", nPackages, nNodePerPkg,
717 nTilePerNode, nCorePerTile, __kmp_nThreadsPerCore,
720 KMP_INFORM(TopologyExtraNode,
"KMP_AFFINITY", nPackages, nNodePerPkg,
721 nCorePerNode, __kmp_nThreadsPerCore, __kmp_ncores);
722 nPUs *= (nNodePerPkg * nCorePerNode);
725 if (__kmp_tile_depth) {
726 KMP_INFORM(TopologyExtraTile,
"KMP_AFFINITY", nPackages, nTilePerPkg,
727 nCorePerTile, __kmp_nThreadsPerCore, __kmp_ncores);
730 __kmp_str_buf_init(&buf);
731 __kmp_str_buf_print(&buf,
"%d", nPackages);
732 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, nCoresPerPkg,
733 __kmp_nThreadsPerCore, __kmp_ncores);
734 __kmp_str_buf_free(&buf);
739 if (__kmp_affinity_type == affinity_none) {
741 KMP_CPU_FREE(oldMask);
745 int depth_full = depth;
748 depth = __kmp_affinity_remove_radix_one_levels(retval, nActiveThreads, depth,
750 KMP_DEBUG_ASSERT(__kmp_affinity_gran != affinity_gran_default);
751 if (__kmp_affinity_gran_levels < 0) {
754 __kmp_affinity_gran_levels = 0;
755 if (__kmp_affinity_gran > affinity_gran_thread) {
756 for (
int i = 1; i <= depth_full; ++i) {
757 if (__kmp_affinity_gran <= i)
759 if (levels[depth_full - i] > 0)
760 __kmp_affinity_gran_levels++;
763 if (__kmp_affinity_gran > affinity_gran_package)
764 __kmp_affinity_gran_levels++;
767 if (__kmp_affinity_verbose)
768 __kmp_affinity_print_hwloc_tp(retval, nActiveThreads, depth, levels);
770 KMP_CPU_FREE(oldMask);
771 *address2os = retval;
774 #endif // KMP_USE_HWLOC 779 static int __kmp_affinity_create_flat_map(AddrUnsPair **address2os,
780 kmp_i18n_id_t *
const msg_id) {
782 *msg_id = kmp_i18n_null;
787 if (!KMP_AFFINITY_CAPABLE()) {
788 KMP_ASSERT(__kmp_affinity_type == affinity_none);
789 __kmp_ncores = nPackages = __kmp_xproc;
790 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
791 if (__kmp_affinity_verbose) {
792 KMP_INFORM(AffFlatTopology,
"KMP_AFFINITY");
793 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
794 KMP_INFORM(Uniform,
"KMP_AFFINITY");
795 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
796 __kmp_nThreadsPerCore, __kmp_ncores);
805 __kmp_ncores = nPackages = __kmp_avail_proc;
806 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
807 if (__kmp_affinity_verbose) {
808 char buf[KMP_AFFIN_MASK_PRINT_LEN];
809 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
810 __kmp_affin_fullMask);
812 KMP_INFORM(AffCapableUseFlat,
"KMP_AFFINITY");
813 if (__kmp_affinity_respect_mask) {
814 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
816 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
818 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
819 KMP_INFORM(Uniform,
"KMP_AFFINITY");
820 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
821 __kmp_nThreadsPerCore, __kmp_ncores);
823 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
824 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
825 if (__kmp_affinity_type == affinity_none) {
828 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
829 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask))
831 __kmp_pu_os_idx[avail_ct++] = i;
838 (AddrUnsPair *)__kmp_allocate(
sizeof(**address2os) * __kmp_avail_proc);
841 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
843 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
846 __kmp_pu_os_idx[avail_ct] = i;
849 (*address2os)[avail_ct++] = AddrUnsPair(addr, i);
851 if (__kmp_affinity_verbose) {
852 KMP_INFORM(OSProcToPackage,
"KMP_AFFINITY");
855 if (__kmp_affinity_gran_levels < 0) {
858 if (__kmp_affinity_gran > affinity_gran_package) {
859 __kmp_affinity_gran_levels = 1;
861 __kmp_affinity_gran_levels = 0;
867 #if KMP_GROUP_AFFINITY 873 static int __kmp_affinity_create_proc_group_map(AddrUnsPair **address2os,
874 kmp_i18n_id_t *
const msg_id) {
876 *msg_id = kmp_i18n_null;
880 if (!KMP_AFFINITY_CAPABLE()) {
887 (AddrUnsPair *)__kmp_allocate(
sizeof(**address2os) * __kmp_avail_proc);
888 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
889 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
892 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
894 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
897 __kmp_pu_os_idx[avail_ct] = i;
899 addr.labels[0] = i / (CHAR_BIT *
sizeof(DWORD_PTR));
900 addr.labels[1] = i % (CHAR_BIT *
sizeof(DWORD_PTR));
901 (*address2os)[avail_ct++] = AddrUnsPair(addr, i);
903 if (__kmp_affinity_verbose) {
904 KMP_INFORM(AffOSProcToGroup,
"KMP_AFFINITY", i, addr.labels[0],
909 if (__kmp_affinity_gran_levels < 0) {
910 if (__kmp_affinity_gran == affinity_gran_group) {
911 __kmp_affinity_gran_levels = 1;
912 }
else if ((__kmp_affinity_gran == affinity_gran_fine) ||
913 (__kmp_affinity_gran == affinity_gran_thread)) {
914 __kmp_affinity_gran_levels = 0;
916 const char *gran_str = NULL;
917 if (__kmp_affinity_gran == affinity_gran_core) {
919 }
else if (__kmp_affinity_gran == affinity_gran_package) {
920 gran_str =
"package";
921 }
else if (__kmp_affinity_gran == affinity_gran_node) {
929 __kmp_affinity_gran_levels = 0;
937 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 939 static int __kmp_cpuid_mask_width(
int count) {
942 while ((1 << r) < count)
947 class apicThreadInfo {
951 unsigned maxCoresPerPkg;
952 unsigned maxThreadsPerPkg;
958 static int __kmp_affinity_cmp_apicThreadInfo_os_id(
const void *a,
960 const apicThreadInfo *aa = (
const apicThreadInfo *)a;
961 const apicThreadInfo *bb = (
const apicThreadInfo *)b;
962 if (aa->osId < bb->osId)
964 if (aa->osId > bb->osId)
969 static int __kmp_affinity_cmp_apicThreadInfo_phys_id(
const void *a,
971 const apicThreadInfo *aa = (
const apicThreadInfo *)a;
972 const apicThreadInfo *bb = (
const apicThreadInfo *)b;
973 if (aa->pkgId < bb->pkgId)
975 if (aa->pkgId > bb->pkgId)
977 if (aa->coreId < bb->coreId)
979 if (aa->coreId > bb->coreId)
981 if (aa->threadId < bb->threadId)
983 if (aa->threadId > bb->threadId)
992 static int __kmp_affinity_create_apicid_map(AddrUnsPair **address2os,
993 kmp_i18n_id_t *
const msg_id) {
997 *msg_id = kmp_i18n_null;
1000 __kmp_x86_cpuid(0, 0, &buf);
1002 *msg_id = kmp_i18n_str_NoLeaf4Support;
1011 if (!KMP_AFFINITY_CAPABLE()) {
1014 KMP_ASSERT(__kmp_affinity_type == affinity_none);
1020 __kmp_x86_cpuid(1, 0, &buf);
1021 int maxThreadsPerPkg = (buf.ebx >> 16) & 0xff;
1022 if (maxThreadsPerPkg == 0) {
1023 maxThreadsPerPkg = 1;
1037 __kmp_x86_cpuid(0, 0, &buf);
1039 __kmp_x86_cpuid(4, 0, &buf);
1040 nCoresPerPkg = ((buf.eax >> 26) & 0x3f) + 1;
1058 __kmp_ncores = __kmp_xproc;
1059 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
1060 __kmp_nThreadsPerCore = 1;
1061 if (__kmp_affinity_verbose) {
1062 KMP_INFORM(AffNotCapableUseLocCpuid,
"KMP_AFFINITY");
1063 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1064 if (__kmp_affinity_uniform_topology()) {
1065 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1067 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1069 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1070 __kmp_nThreadsPerCore, __kmp_ncores);
1080 kmp_affin_mask_t *oldMask;
1081 KMP_CPU_ALLOC(oldMask);
1082 KMP_ASSERT(oldMask != NULL);
1083 __kmp_get_system_affinity(oldMask, TRUE);
1111 apicThreadInfo *threadInfo = (apicThreadInfo *)__kmp_allocate(
1112 __kmp_avail_proc *
sizeof(apicThreadInfo));
1113 unsigned nApics = 0;
1114 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
1116 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
1119 KMP_DEBUG_ASSERT((
int)nApics < __kmp_avail_proc);
1121 __kmp_affinity_dispatch->bind_thread(i);
1122 threadInfo[nApics].osId = i;
1125 __kmp_x86_cpuid(1, 0, &buf);
1126 if (((buf.edx >> 9) & 1) == 0) {
1127 __kmp_set_system_affinity(oldMask, TRUE);
1128 __kmp_free(threadInfo);
1129 KMP_CPU_FREE(oldMask);
1130 *msg_id = kmp_i18n_str_ApicNotPresent;
1133 threadInfo[nApics].apicId = (buf.ebx >> 24) & 0xff;
1134 threadInfo[nApics].maxThreadsPerPkg = (buf.ebx >> 16) & 0xff;
1135 if (threadInfo[nApics].maxThreadsPerPkg == 0) {
1136 threadInfo[nApics].maxThreadsPerPkg = 1;
1145 __kmp_x86_cpuid(0, 0, &buf);
1147 __kmp_x86_cpuid(4, 0, &buf);
1148 threadInfo[nApics].maxCoresPerPkg = ((buf.eax >> 26) & 0x3f) + 1;
1150 threadInfo[nApics].maxCoresPerPkg = 1;
1154 int widthCT = __kmp_cpuid_mask_width(threadInfo[nApics].maxThreadsPerPkg);
1155 threadInfo[nApics].pkgId = threadInfo[nApics].apicId >> widthCT;
1157 int widthC = __kmp_cpuid_mask_width(threadInfo[nApics].maxCoresPerPkg);
1158 int widthT = widthCT - widthC;
1163 __kmp_set_system_affinity(oldMask, TRUE);
1164 __kmp_free(threadInfo);
1165 KMP_CPU_FREE(oldMask);
1166 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1170 int maskC = (1 << widthC) - 1;
1171 threadInfo[nApics].coreId = (threadInfo[nApics].apicId >> widthT) & maskC;
1173 int maskT = (1 << widthT) - 1;
1174 threadInfo[nApics].threadId = threadInfo[nApics].apicId & maskT;
1181 __kmp_set_system_affinity(oldMask, TRUE);
1190 KMP_ASSERT(nApics > 0);
1192 __kmp_ncores = nPackages = 1;
1193 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
1194 if (__kmp_affinity_verbose) {
1195 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1196 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1198 KMP_INFORM(AffUseGlobCpuid,
"KMP_AFFINITY");
1199 if (__kmp_affinity_respect_mask) {
1200 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
1202 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
1204 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1205 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1206 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1207 __kmp_nThreadsPerCore, __kmp_ncores);
1210 if (__kmp_affinity_type == affinity_none) {
1211 __kmp_free(threadInfo);
1212 KMP_CPU_FREE(oldMask);
1216 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair));
1218 addr.labels[0] = threadInfo[0].pkgId;
1219 (*address2os)[0] = AddrUnsPair(addr, threadInfo[0].osId);
1221 if (__kmp_affinity_gran_levels < 0) {
1222 __kmp_affinity_gran_levels = 0;
1225 if (__kmp_affinity_verbose) {
1226 __kmp_affinity_print_topology(*address2os, 1, 1, 0, -1, -1);
1229 __kmp_free(threadInfo);
1230 KMP_CPU_FREE(oldMask);
1235 qsort(threadInfo, nApics,
sizeof(*threadInfo),
1236 __kmp_affinity_cmp_apicThreadInfo_phys_id);
1253 __kmp_nThreadsPerCore = 1;
1254 unsigned nCores = 1;
1257 unsigned lastPkgId = threadInfo[0].pkgId;
1258 unsigned coreCt = 1;
1259 unsigned lastCoreId = threadInfo[0].coreId;
1260 unsigned threadCt = 1;
1261 unsigned lastThreadId = threadInfo[0].threadId;
1264 unsigned prevMaxCoresPerPkg = threadInfo[0].maxCoresPerPkg;
1265 unsigned prevMaxThreadsPerPkg = threadInfo[0].maxThreadsPerPkg;
1267 for (i = 1; i < nApics; i++) {
1268 if (threadInfo[i].pkgId != lastPkgId) {
1271 lastPkgId = threadInfo[i].pkgId;
1272 if ((
int)coreCt > nCoresPerPkg)
1273 nCoresPerPkg = coreCt;
1275 lastCoreId = threadInfo[i].coreId;
1276 if ((
int)threadCt > __kmp_nThreadsPerCore)
1277 __kmp_nThreadsPerCore = threadCt;
1279 lastThreadId = threadInfo[i].threadId;
1283 prevMaxCoresPerPkg = threadInfo[i].maxCoresPerPkg;
1284 prevMaxThreadsPerPkg = threadInfo[i].maxThreadsPerPkg;
1288 if (threadInfo[i].coreId != lastCoreId) {
1291 lastCoreId = threadInfo[i].coreId;
1292 if ((
int)threadCt > __kmp_nThreadsPerCore)
1293 __kmp_nThreadsPerCore = threadCt;
1295 lastThreadId = threadInfo[i].threadId;
1296 }
else if (threadInfo[i].threadId != lastThreadId) {
1298 lastThreadId = threadInfo[i].threadId;
1300 __kmp_free(threadInfo);
1301 KMP_CPU_FREE(oldMask);
1302 *msg_id = kmp_i18n_str_LegacyApicIDsNotUnique;
1308 if ((prevMaxCoresPerPkg != threadInfo[i].maxCoresPerPkg) ||
1309 (prevMaxThreadsPerPkg != threadInfo[i].maxThreadsPerPkg)) {
1310 __kmp_free(threadInfo);
1311 KMP_CPU_FREE(oldMask);
1312 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1317 if ((
int)coreCt > nCoresPerPkg)
1318 nCoresPerPkg = coreCt;
1319 if ((
int)threadCt > __kmp_nThreadsPerCore)
1320 __kmp_nThreadsPerCore = threadCt;
1326 __kmp_ncores = nCores;
1327 if (__kmp_affinity_verbose) {
1328 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1329 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1331 KMP_INFORM(AffUseGlobCpuid,
"KMP_AFFINITY");
1332 if (__kmp_affinity_respect_mask) {
1333 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
1335 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
1337 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1338 if (__kmp_affinity_uniform_topology()) {
1339 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1341 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1343 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1344 __kmp_nThreadsPerCore, __kmp_ncores);
1346 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
1347 KMP_DEBUG_ASSERT(nApics == __kmp_avail_proc);
1348 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
1349 for (i = 0; i < nApics; ++i) {
1350 __kmp_pu_os_idx[i] = threadInfo[i].osId;
1352 if (__kmp_affinity_type == affinity_none) {
1353 __kmp_free(threadInfo);
1354 KMP_CPU_FREE(oldMask);
1362 int coreLevel = (nCoresPerPkg <= 1) ? -1 : 1;
1364 (__kmp_nThreadsPerCore <= 1) ? -1 : ((coreLevel >= 0) ? 2 : 1);
1365 unsigned depth = (pkgLevel >= 0) + (coreLevel >= 0) + (threadLevel >= 0);
1367 KMP_ASSERT(depth > 0);
1368 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * nApics);
1370 for (i = 0; i < nApics; ++i) {
1371 Address addr(depth);
1372 unsigned os = threadInfo[i].osId;
1375 if (pkgLevel >= 0) {
1376 addr.labels[d++] = threadInfo[i].pkgId;
1378 if (coreLevel >= 0) {
1379 addr.labels[d++] = threadInfo[i].coreId;
1381 if (threadLevel >= 0) {
1382 addr.labels[d++] = threadInfo[i].threadId;
1384 (*address2os)[i] = AddrUnsPair(addr, os);
1387 if (__kmp_affinity_gran_levels < 0) {
1390 __kmp_affinity_gran_levels = 0;
1391 if ((threadLevel >= 0) && (__kmp_affinity_gran > affinity_gran_thread)) {
1392 __kmp_affinity_gran_levels++;
1394 if ((coreLevel >= 0) && (__kmp_affinity_gran > affinity_gran_core)) {
1395 __kmp_affinity_gran_levels++;
1397 if ((pkgLevel >= 0) && (__kmp_affinity_gran > affinity_gran_package)) {
1398 __kmp_affinity_gran_levels++;
1402 if (__kmp_affinity_verbose) {
1403 __kmp_affinity_print_topology(*address2os, nApics, depth, pkgLevel,
1404 coreLevel, threadLevel);
1407 __kmp_free(threadInfo);
1408 KMP_CPU_FREE(oldMask);
1415 static int __kmp_affinity_create_x2apicid_map(AddrUnsPair **address2os,
1416 kmp_i18n_id_t *
const msg_id) {
1419 *msg_id = kmp_i18n_null;
1422 __kmp_x86_cpuid(0, 0, &buf);
1424 *msg_id = kmp_i18n_str_NoLeaf11Support;
1427 __kmp_x86_cpuid(11, 0, &buf);
1429 *msg_id = kmp_i18n_str_NoLeaf11Support;
1438 int threadLevel = -1;
1441 __kmp_nThreadsPerCore = nCoresPerPkg = nPackages = 1;
1443 for (level = 0;; level++) {
1454 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1457 __kmp_x86_cpuid(11, level, &buf);
1466 int kind = (buf.ecx >> 8) & 0xff;
1469 threadLevel = level;
1472 __kmp_nThreadsPerCore = buf.ebx & 0xffff;
1473 if (__kmp_nThreadsPerCore == 0) {
1474 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1477 }
else if (kind == 2) {
1481 nCoresPerPkg = buf.ebx & 0xffff;
1482 if (nCoresPerPkg == 0) {
1483 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1488 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1491 if (pkgLevel >= 0) {
1495 nPackages = buf.ebx & 0xffff;
1496 if (nPackages == 0) {
1497 *msg_id = kmp_i18n_str_InvalidCpuidInfo;
1508 if (threadLevel >= 0) {
1509 threadLevel = depth - threadLevel - 1;
1511 if (coreLevel >= 0) {
1512 coreLevel = depth - coreLevel - 1;
1514 KMP_DEBUG_ASSERT(pkgLevel >= 0);
1515 pkgLevel = depth - pkgLevel - 1;
1522 if (!KMP_AFFINITY_CAPABLE()) {
1525 KMP_ASSERT(__kmp_affinity_type == affinity_none);
1527 __kmp_ncores = __kmp_xproc / __kmp_nThreadsPerCore;
1528 nPackages = (__kmp_xproc + nCoresPerPkg - 1) / nCoresPerPkg;
1529 if (__kmp_affinity_verbose) {
1530 KMP_INFORM(AffNotCapableUseLocCpuidL11,
"KMP_AFFINITY");
1531 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1532 if (__kmp_affinity_uniform_topology()) {
1533 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1535 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1537 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1538 __kmp_nThreadsPerCore, __kmp_ncores);
1548 kmp_affin_mask_t *oldMask;
1549 KMP_CPU_ALLOC(oldMask);
1550 __kmp_get_system_affinity(oldMask, TRUE);
1553 AddrUnsPair *retval =
1554 (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * __kmp_avail_proc);
1560 KMP_CPU_SET_ITERATE(proc, __kmp_affin_fullMask) {
1562 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
1565 KMP_DEBUG_ASSERT(nApics < __kmp_avail_proc);
1567 __kmp_affinity_dispatch->bind_thread(proc);
1570 Address addr(depth);
1573 for (level = 0; level < depth; level++) {
1574 __kmp_x86_cpuid(11, level, &buf);
1575 unsigned apicId = buf.edx;
1577 if (level != depth - 1) {
1578 KMP_CPU_FREE(oldMask);
1579 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1582 addr.labels[depth - level - 1] = apicId >> prev_shift;
1586 int shift = buf.eax & 0x1f;
1587 int mask = (1 << shift) - 1;
1588 addr.labels[depth - level - 1] = (apicId & mask) >> prev_shift;
1591 if (level != depth) {
1592 KMP_CPU_FREE(oldMask);
1593 *msg_id = kmp_i18n_str_InconsistentCpuidInfo;
1597 retval[nApics] = AddrUnsPair(addr, proc);
1603 __kmp_set_system_affinity(oldMask, TRUE);
1606 KMP_ASSERT(nApics > 0);
1608 __kmp_ncores = nPackages = 1;
1609 __kmp_nThreadsPerCore = nCoresPerPkg = 1;
1610 if (__kmp_affinity_verbose) {
1611 char buf[KMP_AFFIN_MASK_PRINT_LEN];
1612 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1614 KMP_INFORM(AffUseGlobCpuidL11,
"KMP_AFFINITY");
1615 if (__kmp_affinity_respect_mask) {
1616 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
1618 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
1620 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1621 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1622 KMP_INFORM(Topology,
"KMP_AFFINITY", nPackages, nCoresPerPkg,
1623 __kmp_nThreadsPerCore, __kmp_ncores);
1626 if (__kmp_affinity_type == affinity_none) {
1628 KMP_CPU_FREE(oldMask);
1634 addr.labels[0] = retval[0].first.labels[pkgLevel];
1635 retval[0].first = addr;
1637 if (__kmp_affinity_gran_levels < 0) {
1638 __kmp_affinity_gran_levels = 0;
1641 if (__kmp_affinity_verbose) {
1642 __kmp_affinity_print_topology(retval, 1, 1, 0, -1, -1);
1645 *address2os = retval;
1646 KMP_CPU_FREE(oldMask);
1651 qsort(retval, nApics,
sizeof(*retval), __kmp_affinity_cmp_Address_labels);
1654 unsigned *totals = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1655 unsigned *counts = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1656 unsigned *maxCt = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1657 unsigned *last = (
unsigned *)__kmp_allocate(depth *
sizeof(
unsigned));
1658 for (level = 0; level < depth; level++) {
1662 last[level] = retval[0].first.labels[level];
1669 for (proc = 1; (int)proc < nApics; proc++) {
1671 for (level = 0; level < depth; level++) {
1672 if (retval[proc].first.labels[level] != last[level]) {
1674 for (j = level + 1; j < depth; j++) {
1685 last[j] = retval[proc].first.labels[j];
1689 if (counts[level] > maxCt[level]) {
1690 maxCt[level] = counts[level];
1692 last[level] = retval[proc].first.labels[level];
1694 }
else if (level == depth - 1) {
1700 KMP_CPU_FREE(oldMask);
1701 *msg_id = kmp_i18n_str_x2ApicIDsNotUnique;
1711 if (threadLevel >= 0) {
1712 __kmp_nThreadsPerCore = maxCt[threadLevel];
1714 __kmp_nThreadsPerCore = 1;
1716 nPackages = totals[pkgLevel];
1718 if (coreLevel >= 0) {
1719 __kmp_ncores = totals[coreLevel];
1720 nCoresPerPkg = maxCt[coreLevel];
1722 __kmp_ncores = nPackages;
1727 unsigned prod = maxCt[0];
1728 for (level = 1; level < depth; level++) {
1729 prod *= maxCt[level];
1731 bool uniform = (prod == totals[level - 1]);
1734 if (__kmp_affinity_verbose) {
1735 char mask[KMP_AFFIN_MASK_PRINT_LEN];
1736 __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
1738 KMP_INFORM(AffUseGlobCpuidL11,
"KMP_AFFINITY");
1739 if (__kmp_affinity_respect_mask) {
1740 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", mask);
1742 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", mask);
1744 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
1746 KMP_INFORM(Uniform,
"KMP_AFFINITY");
1748 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
1752 __kmp_str_buf_init(&buf);
1754 __kmp_str_buf_print(&buf,
"%d", totals[0]);
1755 for (level = 1; level <= pkgLevel; level++) {
1756 __kmp_str_buf_print(&buf,
" x %d", maxCt[level]);
1758 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, nCoresPerPkg,
1759 __kmp_nThreadsPerCore, __kmp_ncores);
1761 __kmp_str_buf_free(&buf);
1763 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
1764 KMP_DEBUG_ASSERT(nApics == __kmp_avail_proc);
1765 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
1766 for (proc = 0; (int)proc < nApics; ++proc) {
1767 __kmp_pu_os_idx[proc] = retval[proc].second;
1769 if (__kmp_affinity_type == affinity_none) {
1775 KMP_CPU_FREE(oldMask);
1782 for (level = 0; level < depth; level++) {
1783 if ((maxCt[level] == 1) && (level != pkgLevel)) {
1791 if (new_depth != depth) {
1792 AddrUnsPair *new_retval =
1793 (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * nApics);
1794 for (proc = 0; (int)proc < nApics; proc++) {
1795 Address addr(new_depth);
1796 new_retval[proc] = AddrUnsPair(addr, retval[proc].second);
1799 int newPkgLevel = -1;
1800 int newCoreLevel = -1;
1801 int newThreadLevel = -1;
1803 for (level = 0; level < depth; level++) {
1804 if ((maxCt[level] == 1) && (level != pkgLevel)) {
1808 if (level == pkgLevel) {
1809 newPkgLevel = new_level;
1811 if (level == coreLevel) {
1812 newCoreLevel = new_level;
1814 if (level == threadLevel) {
1815 newThreadLevel = new_level;
1817 for (proc = 0; (int)proc < nApics; proc++) {
1818 new_retval[proc].first.labels[new_level] =
1819 retval[proc].first.labels[level];
1825 retval = new_retval;
1827 pkgLevel = newPkgLevel;
1828 coreLevel = newCoreLevel;
1829 threadLevel = newThreadLevel;
1832 if (__kmp_affinity_gran_levels < 0) {
1835 __kmp_affinity_gran_levels = 0;
1836 if ((threadLevel >= 0) && (__kmp_affinity_gran > affinity_gran_thread)) {
1837 __kmp_affinity_gran_levels++;
1839 if ((coreLevel >= 0) && (__kmp_affinity_gran > affinity_gran_core)) {
1840 __kmp_affinity_gran_levels++;
1842 if (__kmp_affinity_gran > affinity_gran_package) {
1843 __kmp_affinity_gran_levels++;
1847 if (__kmp_affinity_verbose) {
1848 __kmp_affinity_print_topology(retval, nApics, depth, pkgLevel, coreLevel,
1856 KMP_CPU_FREE(oldMask);
1857 *address2os = retval;
1864 #define threadIdIndex 1 1865 #define coreIdIndex 2 1866 #define pkgIdIndex 3 1867 #define nodeIdIndex 4 1869 typedef unsigned *ProcCpuInfo;
1870 static unsigned maxIndex = pkgIdIndex;
1872 static int __kmp_affinity_cmp_ProcCpuInfo_os_id(
const void *a,
const void *b) {
1873 const unsigned *aa = (
const unsigned *)a;
1874 const unsigned *bb = (
const unsigned *)b;
1875 if (aa[osIdIndex] < bb[osIdIndex])
1877 if (aa[osIdIndex] > bb[osIdIndex])
1882 static int __kmp_affinity_cmp_ProcCpuInfo_phys_id(
const void *a,
1885 const unsigned *aa = *(
unsigned *
const *)a;
1886 const unsigned *bb = *(
unsigned *
const *)b;
1887 for (i = maxIndex;; i--) {
1900 static int __kmp_affinity_create_cpuinfo_map(AddrUnsPair **address2os,
1902 kmp_i18n_id_t *
const msg_id,
1905 *msg_id = kmp_i18n_null;
1910 unsigned num_records = 0;
1912 buf[
sizeof(buf) - 1] = 1;
1913 if (!fgets(buf,
sizeof(buf), f)) {
1918 char s1[] =
"processor";
1919 if (strncmp(buf, s1,
sizeof(s1) - 1) == 0) {
1926 if (KMP_SSCANF(buf,
"node_%u id", &level) == 1) {
1927 if (nodeIdIndex + level >= maxIndex) {
1928 maxIndex = nodeIdIndex + level;
1936 if (num_records == 0) {
1938 *msg_id = kmp_i18n_str_NoProcRecords;
1941 if (num_records > (
unsigned)__kmp_xproc) {
1943 *msg_id = kmp_i18n_str_TooManyProcRecords;
1952 if (fseek(f, 0, SEEK_SET) != 0) {
1954 *msg_id = kmp_i18n_str_CantRewindCpuinfo;
1960 unsigned **threadInfo =
1961 (
unsigned **)__kmp_allocate((num_records + 1) *
sizeof(
unsigned *));
1963 for (i = 0; i <= num_records; i++) {
1965 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
1968 #define CLEANUP_THREAD_INFO \ 1969 for (i = 0; i <= num_records; i++) { \ 1970 __kmp_free(threadInfo[i]); \ 1972 __kmp_free(threadInfo); 1977 #define INIT_PROC_INFO(p) \ 1978 for (__index = 0; __index <= maxIndex; __index++) { \ 1979 (p)[__index] = UINT_MAX; \ 1982 for (i = 0; i <= num_records; i++) {
1983 INIT_PROC_INFO(threadInfo[i]);
1986 unsigned num_avail = 0;
1993 buf[
sizeof(buf) - 1] = 1;
1994 bool long_line =
false;
1995 if (!fgets(buf,
sizeof(buf), f)) {
2000 for (i = 0; i <= maxIndex; i++) {
2001 if (threadInfo[num_avail][i] != UINT_MAX) {
2009 }
else if (!buf[
sizeof(buf) - 1]) {
2014 #define CHECK_LINE \ 2016 CLEANUP_THREAD_INFO; \ 2017 *msg_id = kmp_i18n_str_LongLineCpuinfo; \ 2023 char s1[] =
"processor";
2024 if (strncmp(buf, s1,
sizeof(s1) - 1) == 0) {
2026 char *p = strchr(buf +
sizeof(s1) - 1,
':');
2028 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2030 if (threadInfo[num_avail][osIdIndex] != UINT_MAX)
2031 #if KMP_ARCH_AARCH64 2040 threadInfo[num_avail][osIdIndex] = val;
2041 #if KMP_OS_LINUX && !(KMP_ARCH_X86 || KMP_ARCH_X86_64) 2045 "/sys/devices/system/cpu/cpu%u/topology/physical_package_id",
2046 threadInfo[num_avail][osIdIndex]);
2047 __kmp_read_from_file(path,
"%u", &threadInfo[num_avail][pkgIdIndex]);
2049 KMP_SNPRINTF(path,
sizeof(path),
2050 "/sys/devices/system/cpu/cpu%u/topology/core_id",
2051 threadInfo[num_avail][osIdIndex]);
2052 __kmp_read_from_file(path,
"%u", &threadInfo[num_avail][coreIdIndex]);
2056 char s2[] =
"physical id";
2057 if (strncmp(buf, s2,
sizeof(s2) - 1) == 0) {
2059 char *p = strchr(buf +
sizeof(s2) - 1,
':');
2061 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2063 if (threadInfo[num_avail][pkgIdIndex] != UINT_MAX)
2065 threadInfo[num_avail][pkgIdIndex] = val;
2068 char s3[] =
"core id";
2069 if (strncmp(buf, s3,
sizeof(s3) - 1) == 0) {
2071 char *p = strchr(buf +
sizeof(s3) - 1,
':');
2073 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2075 if (threadInfo[num_avail][coreIdIndex] != UINT_MAX)
2077 threadInfo[num_avail][coreIdIndex] = val;
2079 #endif // KMP_OS_LINUX && USE_SYSFS_INFO 2081 char s4[] =
"thread id";
2082 if (strncmp(buf, s4,
sizeof(s4) - 1) == 0) {
2084 char *p = strchr(buf +
sizeof(s4) - 1,
':');
2086 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2088 if (threadInfo[num_avail][threadIdIndex] != UINT_MAX)
2090 threadInfo[num_avail][threadIdIndex] = val;
2094 if (KMP_SSCANF(buf,
"node_%u id", &level) == 1) {
2096 char *p = strchr(buf +
sizeof(s4) - 1,
':');
2098 if ((p == NULL) || (KMP_SSCANF(p + 1,
"%u\n", &val) != 1))
2100 KMP_ASSERT(nodeIdIndex + level <= maxIndex);
2101 if (threadInfo[num_avail][nodeIdIndex + level] != UINT_MAX)
2103 threadInfo[num_avail][nodeIdIndex + level] = val;
2110 if ((*buf != 0) && (*buf !=
'\n')) {
2115 while (((ch = fgetc(f)) != EOF) && (ch !=
'\n'))
2123 if ((
int)num_avail == __kmp_xproc) {
2124 CLEANUP_THREAD_INFO;
2125 *msg_id = kmp_i18n_str_TooManyEntries;
2131 if (threadInfo[num_avail][osIdIndex] == UINT_MAX) {
2132 CLEANUP_THREAD_INFO;
2133 *msg_id = kmp_i18n_str_MissingProcField;
2136 if (threadInfo[0][pkgIdIndex] == UINT_MAX) {
2137 CLEANUP_THREAD_INFO;
2138 *msg_id = kmp_i18n_str_MissingPhysicalIDField;
2143 if (!KMP_CPU_ISSET(threadInfo[num_avail][osIdIndex],
2144 __kmp_affin_fullMask)) {
2145 INIT_PROC_INFO(threadInfo[num_avail]);
2152 KMP_ASSERT(num_avail <= num_records);
2153 INIT_PROC_INFO(threadInfo[num_avail]);
2158 CLEANUP_THREAD_INFO;
2159 *msg_id = kmp_i18n_str_MissingValCpuinfo;
2163 CLEANUP_THREAD_INFO;
2164 *msg_id = kmp_i18n_str_DuplicateFieldCpuinfo;
2169 #if KMP_MIC && REDUCE_TEAM_SIZE 2170 unsigned teamSize = 0;
2171 #endif // KMP_MIC && REDUCE_TEAM_SIZE 2182 KMP_ASSERT(num_avail > 0);
2183 KMP_ASSERT(num_avail <= num_records);
2184 if (num_avail == 1) {
2186 __kmp_nThreadsPerCore = nCoresPerPkg = nPackages = 1;
2187 if (__kmp_affinity_verbose) {
2188 if (!KMP_AFFINITY_CAPABLE()) {
2189 KMP_INFORM(AffNotCapableUseCpuinfo,
"KMP_AFFINITY");
2190 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2191 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2193 char buf[KMP_AFFIN_MASK_PRINT_LEN];
2194 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
2195 __kmp_affin_fullMask);
2196 KMP_INFORM(AffCapableUseCpuinfo,
"KMP_AFFINITY");
2197 if (__kmp_affinity_respect_mask) {
2198 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
2200 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
2202 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2203 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2207 __kmp_str_buf_init(&buf);
2208 __kmp_str_buf_print(&buf,
"1");
2209 for (index = maxIndex - 1; index > pkgIdIndex; index--) {
2210 __kmp_str_buf_print(&buf,
" x 1");
2212 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, 1, 1, 1);
2213 __kmp_str_buf_free(&buf);
2216 if (__kmp_affinity_type == affinity_none) {
2217 CLEANUP_THREAD_INFO;
2221 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair));
2223 addr.labels[0] = threadInfo[0][pkgIdIndex];
2224 (*address2os)[0] = AddrUnsPair(addr, threadInfo[0][osIdIndex]);
2226 if (__kmp_affinity_gran_levels < 0) {
2227 __kmp_affinity_gran_levels = 0;
2230 if (__kmp_affinity_verbose) {
2231 __kmp_affinity_print_topology(*address2os, 1, 1, 0, -1, -1);
2234 CLEANUP_THREAD_INFO;
2239 qsort(threadInfo, num_avail,
sizeof(*threadInfo),
2240 __kmp_affinity_cmp_ProcCpuInfo_phys_id);
2252 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2254 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2256 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2258 (
unsigned *)__kmp_allocate((maxIndex + 1) *
sizeof(unsigned));
2260 bool assign_thread_ids =
false;
2261 unsigned threadIdCt;
2264 restart_radix_check:
2268 if (assign_thread_ids) {
2269 if (threadInfo[0][threadIdIndex] == UINT_MAX) {
2270 threadInfo[0][threadIdIndex] = threadIdCt++;
2271 }
else if (threadIdCt <= threadInfo[0][threadIdIndex]) {
2272 threadIdCt = threadInfo[0][threadIdIndex] + 1;
2275 for (index = 0; index <= maxIndex; index++) {
2279 lastId[index] = threadInfo[0][index];
2284 for (i = 1; i < num_avail; i++) {
2287 for (index = maxIndex; index >= threadIdIndex; index--) {
2288 if (assign_thread_ids && (index == threadIdIndex)) {
2290 if (threadInfo[i][threadIdIndex] == UINT_MAX) {
2291 threadInfo[i][threadIdIndex] = threadIdCt++;
2295 else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
2296 threadIdCt = threadInfo[i][threadIdIndex] + 1;
2299 if (threadInfo[i][index] != lastId[index]) {
2304 for (index2 = threadIdIndex; index2 < index; index2++) {
2306 if (counts[index2] > maxCt[index2]) {
2307 maxCt[index2] = counts[index2];
2310 lastId[index2] = threadInfo[i][index2];
2314 lastId[index] = threadInfo[i][index];
2316 if (assign_thread_ids && (index > threadIdIndex)) {
2318 #if KMP_MIC && REDUCE_TEAM_SIZE 2321 teamSize += (threadIdCt <= 2) ? (threadIdCt) : (threadIdCt - 1);
2322 #endif // KMP_MIC && REDUCE_TEAM_SIZE 2328 if (threadInfo[i][threadIdIndex] == UINT_MAX) {
2329 threadInfo[i][threadIdIndex] = threadIdCt++;
2335 else if (threadIdCt <= threadInfo[i][threadIdIndex]) {
2336 threadIdCt = threadInfo[i][threadIdIndex] + 1;
2342 if (index < threadIdIndex) {
2346 if ((threadInfo[i][threadIdIndex] != UINT_MAX) || assign_thread_ids) {
2351 CLEANUP_THREAD_INFO;
2352 *msg_id = kmp_i18n_str_PhysicalIDsNotUnique;
2358 assign_thread_ids =
true;
2359 goto restart_radix_check;
2363 #if KMP_MIC && REDUCE_TEAM_SIZE 2366 teamSize += (threadIdCt <= 2) ? (threadIdCt) : (threadIdCt - 1);
2367 #endif // KMP_MIC && REDUCE_TEAM_SIZE 2369 for (index = threadIdIndex; index <= maxIndex; index++) {
2370 if (counts[index] > maxCt[index]) {
2371 maxCt[index] = counts[index];
2375 __kmp_nThreadsPerCore = maxCt[threadIdIndex];
2376 nCoresPerPkg = maxCt[coreIdIndex];
2377 nPackages = totals[pkgIdIndex];
2380 unsigned prod = totals[maxIndex];
2381 for (index = threadIdIndex; index < maxIndex; index++) {
2382 prod *= maxCt[index];
2384 bool uniform = (prod == totals[threadIdIndex]);
2390 __kmp_ncores = totals[coreIdIndex];
2392 if (__kmp_affinity_verbose) {
2393 if (!KMP_AFFINITY_CAPABLE()) {
2394 KMP_INFORM(AffNotCapableUseCpuinfo,
"KMP_AFFINITY");
2395 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2397 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2399 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
2402 char buf[KMP_AFFIN_MASK_PRINT_LEN];
2403 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
2404 __kmp_affin_fullMask);
2405 KMP_INFORM(AffCapableUseCpuinfo,
"KMP_AFFINITY");
2406 if (__kmp_affinity_respect_mask) {
2407 KMP_INFORM(InitOSProcSetRespect,
"KMP_AFFINITY", buf);
2409 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_AFFINITY", buf);
2411 KMP_INFORM(AvailableOSProc,
"KMP_AFFINITY", __kmp_avail_proc);
2413 KMP_INFORM(Uniform,
"KMP_AFFINITY");
2415 KMP_INFORM(NonUniform,
"KMP_AFFINITY");
2419 __kmp_str_buf_init(&buf);
2421 __kmp_str_buf_print(&buf,
"%d", totals[maxIndex]);
2422 for (index = maxIndex - 1; index >= pkgIdIndex; index--) {
2423 __kmp_str_buf_print(&buf,
" x %d", maxCt[index]);
2425 KMP_INFORM(TopologyExtra,
"KMP_AFFINITY", buf.str, maxCt[coreIdIndex],
2426 maxCt[threadIdIndex], __kmp_ncores);
2428 __kmp_str_buf_free(&buf);
2431 #if KMP_MIC && REDUCE_TEAM_SIZE 2433 if ((__kmp_dflt_team_nth == 0) && (teamSize > 0)) {
2434 __kmp_dflt_team_nth = teamSize;
2435 KA_TRACE(20, (
"__kmp_affinity_create_cpuinfo_map: setting " 2436 "__kmp_dflt_team_nth = %d\n",
2437 __kmp_dflt_team_nth));
2439 #endif // KMP_MIC && REDUCE_TEAM_SIZE 2441 KMP_DEBUG_ASSERT(__kmp_pu_os_idx == NULL);
2442 KMP_DEBUG_ASSERT(num_avail == __kmp_avail_proc);
2443 __kmp_pu_os_idx = (
int *)__kmp_allocate(
sizeof(
int) * __kmp_avail_proc);
2444 for (i = 0; i < num_avail; ++i) {
2445 __kmp_pu_os_idx[i] = threadInfo[i][osIdIndex];
2448 if (__kmp_affinity_type == affinity_none) {
2453 CLEANUP_THREAD_INFO;
2462 bool *inMap = (
bool *)__kmp_allocate((maxIndex + 1) *
sizeof(bool));
2464 for (index = threadIdIndex; index < maxIndex; index++) {
2465 KMP_ASSERT(totals[index] >= totals[index + 1]);
2466 inMap[index] = (totals[index] > totals[index + 1]);
2468 inMap[maxIndex] = (totals[maxIndex] > 1);
2469 inMap[pkgIdIndex] =
true;
2472 for (index = threadIdIndex; index <= maxIndex; index++) {
2477 KMP_ASSERT(depth > 0);
2480 *address2os = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) * num_avail);
2483 int threadLevel = -1;
2485 for (i = 0; i < num_avail; ++i) {
2486 Address addr(depth);
2487 unsigned os = threadInfo[i][osIdIndex];
2491 for (src_index = maxIndex; src_index >= threadIdIndex; src_index--) {
2492 if (!inMap[src_index]) {
2495 addr.labels[dst_index] = threadInfo[i][src_index];
2496 if (src_index == pkgIdIndex) {
2497 pkgLevel = dst_index;
2498 }
else if (src_index == coreIdIndex) {
2499 coreLevel = dst_index;
2500 }
else if (src_index == threadIdIndex) {
2501 threadLevel = dst_index;
2505 (*address2os)[i] = AddrUnsPair(addr, os);
2508 if (__kmp_affinity_gran_levels < 0) {
2512 __kmp_affinity_gran_levels = 0;
2513 for (src_index = threadIdIndex; src_index <= maxIndex; src_index++) {
2514 if (!inMap[src_index]) {
2517 switch (src_index) {
2519 if (__kmp_affinity_gran > affinity_gran_thread) {
2520 __kmp_affinity_gran_levels++;
2525 if (__kmp_affinity_gran > affinity_gran_core) {
2526 __kmp_affinity_gran_levels++;
2531 if (__kmp_affinity_gran > affinity_gran_package) {
2532 __kmp_affinity_gran_levels++;
2539 if (__kmp_affinity_verbose) {
2540 __kmp_affinity_print_topology(*address2os, num_avail, depth, pkgLevel,
2541 coreLevel, threadLevel);
2549 CLEANUP_THREAD_INFO;
2556 static kmp_affin_mask_t *__kmp_create_masks(
unsigned *maxIndex,
2557 unsigned *numUnique,
2558 AddrUnsPair *address2os,
2559 unsigned numAddrs) {
2565 KMP_ASSERT(numAddrs > 0);
2566 depth = address2os[0].first.depth;
2569 for (i = numAddrs - 1;; --i) {
2570 unsigned osId = address2os[i].second;
2571 if (osId > maxOsId) {
2577 kmp_affin_mask_t *osId2Mask;
2578 KMP_CPU_ALLOC_ARRAY(osId2Mask, (maxOsId + 1));
2582 qsort(address2os, numAddrs,
sizeof(*address2os),
2583 __kmp_affinity_cmp_Address_labels);
2585 KMP_ASSERT(__kmp_affinity_gran_levels >= 0);
2586 if (__kmp_affinity_verbose && (__kmp_affinity_gran_levels > 0)) {
2587 KMP_INFORM(ThreadsMigrate,
"KMP_AFFINITY", __kmp_affinity_gran_levels);
2589 if (__kmp_affinity_gran_levels >= (
int)depth) {
2590 if (__kmp_affinity_verbose ||
2591 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
2592 KMP_WARNING(AffThreadsMayMigrate);
2600 unsigned unique = 0;
2602 unsigned leader = 0;
2603 Address *leaderAddr = &(address2os[0].first);
2604 kmp_affin_mask_t *sum;
2605 KMP_CPU_ALLOC_ON_STACK(sum);
2607 KMP_CPU_SET(address2os[0].second, sum);
2608 for (i = 1; i < numAddrs; i++) {
2612 if (leaderAddr->isClose(address2os[i].first, __kmp_affinity_gran_levels)) {
2613 KMP_CPU_SET(address2os[i].second, sum);
2619 for (; j < i; j++) {
2620 unsigned osId = address2os[j].second;
2621 KMP_DEBUG_ASSERT(osId <= maxOsId);
2622 kmp_affin_mask_t *mask = KMP_CPU_INDEX(osId2Mask, osId);
2623 KMP_CPU_COPY(mask, sum);
2624 address2os[j].first.leader = (j == leader);
2630 leaderAddr = &(address2os[i].first);
2632 KMP_CPU_SET(address2os[i].second, sum);
2637 for (; j < i; j++) {
2638 unsigned osId = address2os[j].second;
2639 KMP_DEBUG_ASSERT(osId <= maxOsId);
2640 kmp_affin_mask_t *mask = KMP_CPU_INDEX(osId2Mask, osId);
2641 KMP_CPU_COPY(mask, sum);
2642 address2os[j].first.leader = (j == leader);
2645 KMP_CPU_FREE_FROM_STACK(sum);
2647 *maxIndex = maxOsId;
2648 *numUnique = unique;
2655 static kmp_affin_mask_t *newMasks;
2656 static int numNewMasks;
2657 static int nextNewMask;
2659 #define ADD_MASK(_mask) \ 2661 if (nextNewMask >= numNewMasks) { \ 2664 kmp_affin_mask_t *temp; \ 2665 KMP_CPU_INTERNAL_ALLOC_ARRAY(temp, numNewMasks); \ 2666 for (i = 0; i < numNewMasks / 2; i++) { \ 2667 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i); \ 2668 kmp_affin_mask_t *dest = KMP_CPU_INDEX(temp, i); \ 2669 KMP_CPU_COPY(dest, src); \ 2671 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks / 2); \ 2674 KMP_CPU_COPY(KMP_CPU_INDEX(newMasks, nextNewMask), (_mask)); \ 2678 #define ADD_MASK_OSID(_osId, _osId2Mask, _maxOsId) \ 2680 if (((_osId) > _maxOsId) || \ 2681 (!KMP_CPU_ISSET((_osId), KMP_CPU_INDEX((_osId2Mask), (_osId))))) { \ 2682 if (__kmp_affinity_verbose || \ 2683 (__kmp_affinity_warnings && \ 2684 (__kmp_affinity_type != affinity_none))) { \ 2685 KMP_WARNING(AffIgnoreInvalidProcID, _osId); \ 2688 ADD_MASK(KMP_CPU_INDEX(_osId2Mask, (_osId))); \ 2694 static void __kmp_affinity_process_proclist(kmp_affin_mask_t **out_masks,
2695 unsigned int *out_numMasks,
2696 const char *proclist,
2697 kmp_affin_mask_t *osId2Mask,
2700 const char *scan = proclist;
2701 const char *next = proclist;
2706 KMP_CPU_INTERNAL_ALLOC_ARRAY(newMasks, numNewMasks);
2708 kmp_affin_mask_t *sumMask;
2709 KMP_CPU_ALLOC(sumMask);
2713 int start, end, stride;
2717 if (*next ==
'\0') {
2729 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad proclist");
2731 num = __kmp_str_to_int(scan, *next);
2732 KMP_ASSERT2(num >= 0,
"bad explicit proc list");
2735 if ((num > maxOsId) ||
2736 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
2737 if (__kmp_affinity_verbose ||
2738 (__kmp_affinity_warnings &&
2739 (__kmp_affinity_type != affinity_none))) {
2740 KMP_WARNING(AffIgnoreInvalidProcID, num);
2742 KMP_CPU_ZERO(sumMask);
2744 KMP_CPU_COPY(sumMask, KMP_CPU_INDEX(osId2Mask, num));
2764 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2767 num = __kmp_str_to_int(scan, *next);
2768 KMP_ASSERT2(num >= 0,
"bad explicit proc list");
2771 if ((num > maxOsId) ||
2772 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
2773 if (__kmp_affinity_verbose ||
2774 (__kmp_affinity_warnings &&
2775 (__kmp_affinity_type != affinity_none))) {
2776 KMP_WARNING(AffIgnoreInvalidProcID, num);
2779 KMP_CPU_UNION(sumMask, KMP_CPU_INDEX(osId2Mask, num));
2796 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2798 start = __kmp_str_to_int(scan, *next);
2799 KMP_ASSERT2(start >= 0,
"bad explicit proc list");
2804 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2818 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2820 end = __kmp_str_to_int(scan, *next);
2821 KMP_ASSERT2(end >= 0,
"bad explicit proc list");
2838 KMP_ASSERT2((*next >=
'0') && (*next <=
'9'),
"bad explicit proc list");
2840 stride = __kmp_str_to_int(scan, *next);
2841 KMP_ASSERT2(stride >= 0,
"bad explicit proc list");
2846 KMP_ASSERT2(stride != 0,
"bad explicit proc list");
2848 KMP_ASSERT2(start <= end,
"bad explicit proc list");
2850 KMP_ASSERT2(start >= end,
"bad explicit proc list");
2852 KMP_ASSERT2((end - start) / stride <= 65536,
"bad explicit proc list");
2857 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2859 }
while (start <= end);
2862 ADD_MASK_OSID(start, osId2Mask, maxOsId);
2864 }
while (start >= end);
2875 *out_numMasks = nextNewMask;
2876 if (nextNewMask == 0) {
2878 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
2881 KMP_CPU_ALLOC_ARRAY((*out_masks), nextNewMask);
2882 for (i = 0; i < nextNewMask; i++) {
2883 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i);
2884 kmp_affin_mask_t *dest = KMP_CPU_INDEX((*out_masks), i);
2885 KMP_CPU_COPY(dest, src);
2887 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
2888 KMP_CPU_FREE(sumMask);
2914 static void __kmp_process_subplace_list(
const char **scan,
2915 kmp_affin_mask_t *osId2Mask,
2916 int maxOsId, kmp_affin_mask_t *tempMask,
2921 int start, count, stride, i;
2925 KMP_ASSERT2((**scan >=
'0') && (**scan <=
'9'),
"bad explicit places list");
2928 start = __kmp_str_to_int(*scan, *next);
2929 KMP_ASSERT(start >= 0);
2934 if (**scan ==
'}' || **scan ==
',') {
2935 if ((start > maxOsId) ||
2936 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
2937 if (__kmp_affinity_verbose ||
2938 (__kmp_affinity_warnings &&
2939 (__kmp_affinity_type != affinity_none))) {
2940 KMP_WARNING(AffIgnoreInvalidProcID, start);
2943 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
2946 if (**scan ==
'}') {
2952 KMP_ASSERT2(**scan ==
':',
"bad explicit places list");
2957 KMP_ASSERT2((**scan >=
'0') && (**scan <=
'9'),
"bad explicit places list");
2960 count = __kmp_str_to_int(*scan, *next);
2961 KMP_ASSERT(count >= 0);
2966 if (**scan ==
'}' || **scan ==
',') {
2967 for (i = 0; i < count; i++) {
2968 if ((start > maxOsId) ||
2969 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
2970 if (__kmp_affinity_verbose ||
2971 (__kmp_affinity_warnings &&
2972 (__kmp_affinity_type != affinity_none))) {
2973 KMP_WARNING(AffIgnoreInvalidProcID, start);
2977 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
2982 if (**scan ==
'}') {
2988 KMP_ASSERT2(**scan ==
':',
"bad explicit places list");
2995 if (**scan ==
'+') {
2999 if (**scan ==
'-') {
3007 KMP_ASSERT2((**scan >=
'0') && (**scan <=
'9'),
"bad explicit places list");
3010 stride = __kmp_str_to_int(*scan, *next);
3011 KMP_ASSERT(stride >= 0);
3017 if (**scan ==
'}' || **scan ==
',') {
3018 for (i = 0; i < count; i++) {
3019 if ((start > maxOsId) ||
3020 (!KMP_CPU_ISSET(start, KMP_CPU_INDEX(osId2Mask, start)))) {
3021 if (__kmp_affinity_verbose ||
3022 (__kmp_affinity_warnings &&
3023 (__kmp_affinity_type != affinity_none))) {
3024 KMP_WARNING(AffIgnoreInvalidProcID, start);
3028 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, start));
3033 if (**scan ==
'}') {
3040 KMP_ASSERT2(0,
"bad explicit places list");
3044 static void __kmp_process_place(
const char **scan, kmp_affin_mask_t *osId2Mask,
3045 int maxOsId, kmp_affin_mask_t *tempMask,
3051 if (**scan ==
'{') {
3053 __kmp_process_subplace_list(scan, osId2Mask, maxOsId, tempMask, setSize);
3054 KMP_ASSERT2(**scan ==
'}',
"bad explicit places list");
3056 }
else if (**scan ==
'!') {
3058 __kmp_process_place(scan, osId2Mask, maxOsId, tempMask, setSize);
3059 KMP_CPU_COMPLEMENT(maxOsId, tempMask);
3060 }
else if ((**scan >=
'0') && (**scan <=
'9')) {
3063 int num = __kmp_str_to_int(*scan, *next);
3064 KMP_ASSERT(num >= 0);
3065 if ((num > maxOsId) ||
3066 (!KMP_CPU_ISSET(num, KMP_CPU_INDEX(osId2Mask, num)))) {
3067 if (__kmp_affinity_verbose ||
3068 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
3069 KMP_WARNING(AffIgnoreInvalidProcID, num);
3072 KMP_CPU_UNION(tempMask, KMP_CPU_INDEX(osId2Mask, num));
3077 KMP_ASSERT2(0,
"bad explicit places list");
3082 void __kmp_affinity_process_placelist(kmp_affin_mask_t **out_masks,
3083 unsigned int *out_numMasks,
3084 const char *placelist,
3085 kmp_affin_mask_t *osId2Mask,
3087 int i, j, count, stride, sign;
3088 const char *scan = placelist;
3089 const char *next = placelist;
3092 KMP_CPU_INTERNAL_ALLOC_ARRAY(newMasks, numNewMasks);
3098 kmp_affin_mask_t *tempMask;
3099 kmp_affin_mask_t *previousMask;
3100 KMP_CPU_ALLOC(tempMask);
3101 KMP_CPU_ZERO(tempMask);
3102 KMP_CPU_ALLOC(previousMask);
3103 KMP_CPU_ZERO(previousMask);
3107 __kmp_process_place(&scan, osId2Mask, maxOsId, tempMask, &setSize);
3111 if (*scan ==
'\0' || *scan ==
',') {
3115 KMP_CPU_ZERO(tempMask);
3117 if (*scan ==
'\0') {
3124 KMP_ASSERT2(*scan ==
':',
"bad explicit places list");
3129 KMP_ASSERT2((*scan >=
'0') && (*scan <=
'9'),
"bad explicit places list");
3132 count = __kmp_str_to_int(scan, *next);
3133 KMP_ASSERT(count >= 0);
3138 if (*scan ==
'\0' || *scan ==
',') {
3141 KMP_ASSERT2(*scan ==
':',
"bad explicit places list");
3160 KMP_ASSERT2((*scan >=
'0') && (*scan <=
'9'),
"bad explicit places list");
3163 stride = __kmp_str_to_int(scan, *next);
3164 KMP_DEBUG_ASSERT(stride >= 0);
3170 for (i = 0; i < count; i++) {
3175 KMP_CPU_COPY(previousMask, tempMask);
3176 ADD_MASK(previousMask);
3177 KMP_CPU_ZERO(tempMask);
3179 KMP_CPU_SET_ITERATE(j, previousMask) {
3180 if (!KMP_CPU_ISSET(j, previousMask)) {
3183 if ((j + stride > maxOsId) || (j + stride < 0) ||
3184 (!KMP_CPU_ISSET(j, __kmp_affin_fullMask)) ||
3185 (!KMP_CPU_ISSET(j + stride,
3186 KMP_CPU_INDEX(osId2Mask, j + stride)))) {
3187 if ((__kmp_affinity_verbose ||
3188 (__kmp_affinity_warnings &&
3189 (__kmp_affinity_type != affinity_none))) &&
3191 KMP_WARNING(AffIgnoreInvalidProcID, j + stride);
3195 KMP_CPU_SET(j + stride, tempMask);
3199 KMP_CPU_ZERO(tempMask);
3204 if (*scan ==
'\0') {
3212 KMP_ASSERT2(0,
"bad explicit places list");
3215 *out_numMasks = nextNewMask;
3216 if (nextNewMask == 0) {
3218 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3221 KMP_CPU_ALLOC_ARRAY((*out_masks), nextNewMask);
3222 KMP_CPU_FREE(tempMask);
3223 KMP_CPU_FREE(previousMask);
3224 for (i = 0; i < nextNewMask; i++) {
3225 kmp_affin_mask_t *src = KMP_CPU_INDEX(newMasks, i);
3226 kmp_affin_mask_t *dest = KMP_CPU_INDEX((*out_masks), i);
3227 KMP_CPU_COPY(dest, src);
3229 KMP_CPU_INTERNAL_FREE_ARRAY(newMasks, numNewMasks);
3235 #undef ADD_MASK_OSID 3238 static int __kmp_hwloc_skip_PUs_obj(hwloc_topology_t t, hwloc_obj_t o) {
3241 hwloc_obj_t hT = NULL;
3242 int N = __kmp_hwloc_count_children_by_type(t, o, HWLOC_OBJ_PU, &hT);
3243 for (
int i = 0; i < N; ++i) {
3244 KMP_DEBUG_ASSERT(hT);
3245 unsigned idx = hT->os_index;
3246 if (KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3247 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3248 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3251 hT = hwloc_get_next_obj_by_type(t, HWLOC_OBJ_PU, hT);
3256 static int __kmp_hwloc_obj_has_PUs(hwloc_topology_t t, hwloc_obj_t o) {
3258 hwloc_obj_t hT = NULL;
3259 int N = __kmp_hwloc_count_children_by_type(t, o, HWLOC_OBJ_PU, &hT);
3260 for (
int i = 0; i < N; ++i) {
3261 KMP_DEBUG_ASSERT(hT);
3262 unsigned idx = hT->os_index;
3263 if (KMP_CPU_ISSET(idx, __kmp_affin_fullMask))
3265 hT = hwloc_get_next_obj_by_type(t, HWLOC_OBJ_PU, hT);
3269 #endif // KMP_USE_HWLOC 3271 static void __kmp_apply_thread_places(AddrUnsPair **pAddr,
int depth) {
3272 AddrUnsPair *newAddr;
3273 if (__kmp_hws_requested == 0)
3276 if (__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
3280 hwloc_topology_t tp = __kmp_hwloc_topology;
3281 int nS = 0, nN = 0, nL = 0, nC = 0,
3283 int nCr = 0, nTr = 0;
3284 int nPkg = 0, nCo = 0, n_new = 0, n_old = 0, nCpP = 0, nTpC = 0;
3285 hwloc_obj_t hT, hC, hL, hN, hS;
3289 int numa_support = 0, tile_support = 0;
3290 if (__kmp_pu_os_idx)
3291 hT = hwloc_get_pu_obj_by_os_index(tp,
3292 __kmp_pu_os_idx[__kmp_avail_proc - 1]);
3294 hT = hwloc_get_obj_by_type(tp, HWLOC_OBJ_PU, __kmp_avail_proc - 1);
3296 KMP_WARNING(AffHWSubsetUnsupported);
3300 hN = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hT);
3301 hS = hwloc_get_ancestor_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hT);
3302 if (hN != NULL && hN->depth > hS->depth) {
3304 }
else if (__kmp_hws_node.num > 0) {
3306 KMP_WARNING(AffHWSubsetUnsupported);
3310 L2depth = hwloc_get_cache_type_depth(tp, 2, HWLOC_OBJ_CACHE_UNIFIED);
3311 hL = hwloc_get_ancestor_obj_by_depth(tp, L2depth, hT);
3313 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC) > 1) {
3315 }
else if (__kmp_hws_tile.num > 0) {
3316 if (__kmp_hws_core.num == 0) {
3317 __kmp_hws_core = __kmp_hws_tile;
3318 __kmp_hws_tile.num = 0;
3321 KMP_WARNING(AffHWSubsetInvalid);
3328 if (__kmp_hws_socket.num == 0)
3329 __kmp_hws_socket.num = nPackages;
3330 if (__kmp_hws_socket.offset >= nPackages) {
3331 KMP_WARNING(AffHWSubsetManySockets);
3336 int NN = __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_NUMANODE,
3338 if (__kmp_hws_node.num == 0)
3339 __kmp_hws_node.num = NN;
3340 if (__kmp_hws_node.offset >= NN) {
3341 KMP_WARNING(AffHWSubsetManyNodes);
3346 int NL = __kmp_hwloc_count_children_by_depth(tp, hN, L2depth, &hL);
3347 if (__kmp_hws_tile.num == 0) {
3348 __kmp_hws_tile.num = NL + 1;
3350 if (__kmp_hws_tile.offset >= NL) {
3351 KMP_WARNING(AffHWSubsetManyTiles);
3354 int NC = __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE,
3356 if (__kmp_hws_core.num == 0)
3357 __kmp_hws_core.num = NC;
3358 if (__kmp_hws_core.offset >= NC) {
3359 KMP_WARNING(AffHWSubsetManyCores);
3363 int NC = __kmp_hwloc_count_children_by_type(tp, hN, HWLOC_OBJ_CORE,
3365 if (__kmp_hws_core.num == 0)
3366 __kmp_hws_core.num = NC;
3367 if (__kmp_hws_core.offset >= NC) {
3368 KMP_WARNING(AffHWSubsetManyCores);
3375 int NL = __kmp_hwloc_count_children_by_depth(tp, hS, L2depth, &hL);
3376 if (__kmp_hws_tile.num == 0)
3377 __kmp_hws_tile.num = NL;
3378 if (__kmp_hws_tile.offset >= NL) {
3379 KMP_WARNING(AffHWSubsetManyTiles);
3382 int NC = __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE,
3384 if (__kmp_hws_core.num == 0)
3385 __kmp_hws_core.num = NC;
3386 if (__kmp_hws_core.offset >= NC) {
3387 KMP_WARNING(AffHWSubsetManyCores);
3391 int NC = __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_CORE,
3393 if (__kmp_hws_core.num == 0)
3394 __kmp_hws_core.num = NC;
3395 if (__kmp_hws_core.offset >= NC) {
3396 KMP_WARNING(AffHWSubsetManyCores);
3401 if (__kmp_hws_proc.num == 0)
3402 __kmp_hws_proc.num = __kmp_nThreadsPerCore;
3403 if (__kmp_hws_proc.offset >= __kmp_nThreadsPerCore) {
3404 KMP_WARNING(AffHWSubsetManyProcs);
3410 newAddr = (AddrUnsPair *)__kmp_allocate(
sizeof(AddrUnsPair) *
3414 int NP = hwloc_get_nbobjs_by_type(tp, HWLOC_OBJ_PACKAGE);
3415 for (
int s = 0; s < NP; ++s) {
3417 hS = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PACKAGE, hS);
3418 if (!__kmp_hwloc_obj_has_PUs(tp, hS))
3421 if (nS <= __kmp_hws_socket.offset ||
3422 nS > __kmp_hws_socket.num + __kmp_hws_socket.offset) {
3423 n_old += __kmp_hwloc_skip_PUs_obj(tp, hS);
3434 __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_NUMANODE, &hN);
3435 for (
int n = 0; n < NN; ++n) {
3437 if (!__kmp_hwloc_obj_has_PUs(tp, hN)) {
3438 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3442 if (nN <= __kmp_hws_node.offset ||
3443 nN > __kmp_hws_node.num + __kmp_hws_node.offset) {
3445 n_old += __kmp_hwloc_skip_PUs_obj(tp, hN);
3446 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3453 int NL = __kmp_hwloc_count_children_by_depth(tp, hN, L2depth, &hL);
3454 for (
int l = 0; l < NL; ++l) {
3456 if (!__kmp_hwloc_obj_has_PUs(tp, hL)) {
3457 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3461 if (nL <= __kmp_hws_tile.offset ||
3462 nL > __kmp_hws_tile.num + __kmp_hws_tile.offset) {
3464 n_old += __kmp_hwloc_skip_PUs_obj(tp, hL);
3465 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3472 int NC = __kmp_hwloc_count_children_by_type(tp, hL,
3473 HWLOC_OBJ_CORE, &hC);
3474 for (
int c = 0; c < NC; ++c) {
3476 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3477 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3481 if (nC <= __kmp_hws_core.offset ||
3482 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3484 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3485 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3493 int NT = __kmp_hwloc_count_children_by_type(tp, hC,
3495 for (
int t = 0; t < NT; ++t) {
3498 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3499 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3503 if (nT <= __kmp_hws_proc.offset ||
3504 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3506 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3508 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3509 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3514 newAddr[n_new] = (*pAddr)[n_old];
3517 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3525 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3527 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3535 __kmp_hwloc_count_children_by_type(tp, hN, HWLOC_OBJ_CORE, &hC);
3536 for (
int c = 0; c < NC; ++c) {
3538 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3539 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3543 if (nC <= __kmp_hws_core.offset ||
3544 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3546 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3547 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3555 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3556 for (
int t = 0; t < NT; ++t) {
3559 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3560 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3564 if (nT <= __kmp_hws_proc.offset ||
3565 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3567 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3569 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3570 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3575 newAddr[n_new] = (*pAddr)[n_old];
3578 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3586 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3589 hN = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_NUMANODE, hN);
3597 int NL = __kmp_hwloc_count_children_by_depth(tp, hS, L2depth, &hL);
3598 for (
int l = 0; l < NL; ++l) {
3600 if (!__kmp_hwloc_obj_has_PUs(tp, hL)) {
3601 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3605 if (nL <= __kmp_hws_tile.offset ||
3606 nL > __kmp_hws_tile.num + __kmp_hws_tile.offset) {
3608 n_old += __kmp_hwloc_skip_PUs_obj(tp, hL);
3609 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3617 __kmp_hwloc_count_children_by_type(tp, hL, HWLOC_OBJ_CORE, &hC);
3618 for (
int c = 0; c < NC; ++c) {
3620 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3621 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3625 if (nC <= __kmp_hws_core.offset ||
3626 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3628 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3629 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3638 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3639 for (
int t = 0; t < NT; ++t) {
3642 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3643 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3647 if (nT <= __kmp_hws_proc.offset ||
3648 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3650 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3652 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3653 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3658 newAddr[n_new] = (*pAddr)[n_old];
3661 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3669 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3671 hL = hwloc_get_next_obj_by_depth(tp, L2depth, hL);
3679 __kmp_hwloc_count_children_by_type(tp, hS, HWLOC_OBJ_CORE, &hC);
3680 for (
int c = 0; c < NC; ++c) {
3682 if (!__kmp_hwloc_obj_has_PUs(tp, hC)) {
3683 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3687 if (nC <= __kmp_hws_core.offset ||
3688 nC > __kmp_hws_core.num + __kmp_hws_core.offset) {
3690 n_old += __kmp_hwloc_skip_PUs_obj(tp, hC);
3691 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3700 __kmp_hwloc_count_children_by_type(tp, hC, HWLOC_OBJ_PU, &hT);
3701 for (
int t = 0; t < NT; ++t) {
3704 if (!KMP_CPU_ISSET(idx, __kmp_affin_fullMask)) {
3705 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3709 if (nT <= __kmp_hws_proc.offset ||
3710 nT > __kmp_hws_proc.num + __kmp_hws_proc.offset) {
3712 KMP_CPU_CLR(idx, __kmp_affin_fullMask);
3714 KC_TRACE(200, (
"KMP_HW_SUBSET: skipped proc %d\n", idx));
3715 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3720 newAddr[n_new] = (*pAddr)[n_old];
3723 hT = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_PU, hT);
3731 hC = hwloc_get_next_obj_by_type(tp, HWLOC_OBJ_CORE, hC);
3743 KMP_DEBUG_ASSERT(n_old == __kmp_avail_proc);
3744 KMP_DEBUG_ASSERT(nPkg > 0);
3745 KMP_DEBUG_ASSERT(nCpP > 0);
3746 KMP_DEBUG_ASSERT(nTpC > 0);
3747 KMP_DEBUG_ASSERT(nCo > 0);
3748 KMP_DEBUG_ASSERT(nPkg <= nPackages);
3749 KMP_DEBUG_ASSERT(nCpP <= nCoresPerPkg);
3750 KMP_DEBUG_ASSERT(nTpC <= __kmp_nThreadsPerCore);
3751 KMP_DEBUG_ASSERT(nCo <= __kmp_ncores);
3754 nCoresPerPkg = nCpP;
3755 __kmp_nThreadsPerCore = nTpC;
3756 __kmp_avail_proc = n_new;
3760 #endif // KMP_USE_HWLOC 3762 int n_old = 0, n_new = 0, proc_num = 0;
3763 if (__kmp_hws_node.num > 0 || __kmp_hws_tile.num > 0) {
3764 KMP_WARNING(AffHWSubsetNoHWLOC);
3767 if (__kmp_hws_socket.num == 0)
3768 __kmp_hws_socket.num = nPackages;
3769 if (__kmp_hws_core.num == 0)
3770 __kmp_hws_core.num = nCoresPerPkg;
3771 if (__kmp_hws_proc.num == 0 || __kmp_hws_proc.num > __kmp_nThreadsPerCore)
3772 __kmp_hws_proc.num = __kmp_nThreadsPerCore;
3773 if (!__kmp_affinity_uniform_topology()) {
3774 KMP_WARNING(AffHWSubsetNonUniform);
3778 KMP_WARNING(AffHWSubsetNonThreeLevel);
3781 if (__kmp_hws_socket.offset + __kmp_hws_socket.num > nPackages) {
3782 KMP_WARNING(AffHWSubsetManySockets);
3785 if (__kmp_hws_core.offset + __kmp_hws_core.num > nCoresPerPkg) {
3786 KMP_WARNING(AffHWSubsetManyCores);
3791 newAddr = (AddrUnsPair *)__kmp_allocate(
3792 sizeof(AddrUnsPair) * __kmp_hws_socket.num * __kmp_hws_core.num *
3793 __kmp_hws_proc.num);
3794 for (
int i = 0; i < nPackages; ++i) {
3795 if (i < __kmp_hws_socket.offset ||
3796 i >= __kmp_hws_socket.offset + __kmp_hws_socket.num) {
3798 n_old += nCoresPerPkg * __kmp_nThreadsPerCore;
3799 if (__kmp_pu_os_idx != NULL) {
3801 for (
int j = 0; j < nCoresPerPkg; ++j) {
3802 for (
int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3803 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3810 for (
int j = 0; j < nCoresPerPkg; ++j) {
3811 if (j < __kmp_hws_core.offset ||
3812 j >= __kmp_hws_core.offset +
3813 __kmp_hws_core.num) {
3814 n_old += __kmp_nThreadsPerCore;
3815 if (__kmp_pu_os_idx != NULL) {
3816 for (
int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3817 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3823 for (
int k = 0; k < __kmp_nThreadsPerCore; ++k) {
3824 if (k < __kmp_hws_proc.num) {
3826 newAddr[n_new] = (*pAddr)[n_old];
3829 if (__kmp_pu_os_idx != NULL)
3830 KMP_CPU_CLR(__kmp_pu_os_idx[proc_num], __kmp_affin_fullMask);
3839 KMP_DEBUG_ASSERT(n_old == nPackages * nCoresPerPkg * __kmp_nThreadsPerCore);
3840 KMP_DEBUG_ASSERT(n_new ==
3841 __kmp_hws_socket.num * __kmp_hws_core.num *
3842 __kmp_hws_proc.num);
3843 nPackages = __kmp_hws_socket.num;
3844 nCoresPerPkg = __kmp_hws_core.num;
3845 __kmp_nThreadsPerCore = __kmp_hws_proc.num;
3846 __kmp_avail_proc = n_new;
3847 __kmp_ncores = nPackages * __kmp_hws_core.num;
3853 if (__kmp_affinity_verbose) {
3854 char m[KMP_AFFIN_MASK_PRINT_LEN];
3855 __kmp_affinity_print_mask(m, KMP_AFFIN_MASK_PRINT_LEN,
3856 __kmp_affin_fullMask);
3857 if (__kmp_affinity_respect_mask) {
3858 KMP_INFORM(InitOSProcSetRespect,
"KMP_HW_SUBSET", m);
3860 KMP_INFORM(InitOSProcSetNotRespect,
"KMP_HW_SUBSET", m);
3862 KMP_INFORM(AvailableOSProc,
"KMP_HW_SUBSET", __kmp_avail_proc);
3864 __kmp_str_buf_init(&buf);
3865 __kmp_str_buf_print(&buf,
"%d", nPackages);
3866 KMP_INFORM(TopologyExtra,
"KMP_HW_SUBSET", buf.str, nCoresPerPkg,
3867 __kmp_nThreadsPerCore, __kmp_ncores);
3868 __kmp_str_buf_free(&buf);
3871 if (__kmp_pu_os_idx != NULL) {
3872 __kmp_free(__kmp_pu_os_idx);
3873 __kmp_pu_os_idx = NULL;
3879 static int __kmp_affinity_find_core_level(
const AddrUnsPair *address2os,
3880 int nprocs,
int bottom_level) {
3883 for (
int i = 0; i < nprocs; i++) {
3884 for (
int j = bottom_level; j > 0; j--) {
3885 if (address2os[i].first.labels[j] > 0) {
3886 if (core_level < (j - 1)) {
3896 static int __kmp_affinity_compute_ncores(
const AddrUnsPair *address2os,
3897 int nprocs,
int bottom_level,
3903 for (i = 0; i < nprocs; i++) {
3904 for (j = bottom_level; j > core_level; j--) {
3905 if ((i + 1) < nprocs) {
3906 if (address2os[i + 1].first.labels[j] > 0) {
3911 if (j == core_level) {
3915 if (j > core_level) {
3924 static int __kmp_affinity_find_core(
const AddrUnsPair *address2os,
int proc,
3925 int bottom_level,
int core_level) {
3926 return __kmp_affinity_compute_ncores(address2os, proc + 1, bottom_level,
3933 static int __kmp_affinity_max_proc_per_core(
const AddrUnsPair *address2os,
3934 int nprocs,
int bottom_level,
3936 int maxprocpercore = 0;
3938 if (core_level < bottom_level) {
3939 for (
int i = 0; i < nprocs; i++) {
3940 int percore = address2os[i].first.labels[core_level + 1] + 1;
3942 if (percore > maxprocpercore) {
3943 maxprocpercore = percore;
3949 return maxprocpercore;
3952 static AddrUnsPair *address2os = NULL;
3953 static int *procarr = NULL;
3954 static int __kmp_aff_depth = 0;
3956 #define KMP_EXIT_AFF_NONE \ 3957 KMP_ASSERT(__kmp_affinity_type == affinity_none); \ 3958 KMP_ASSERT(address2os == NULL); \ 3959 __kmp_apply_thread_places(NULL, 0); \ 3962 static int __kmp_affinity_cmp_Address_child_num(
const void *a,
const void *b) {
3963 const Address *aa = &(((
const AddrUnsPair *)a)->first);
3964 const Address *bb = &(((
const AddrUnsPair *)b)->first);
3965 unsigned depth = aa->depth;
3967 KMP_DEBUG_ASSERT(depth == bb->depth);
3968 KMP_DEBUG_ASSERT((
unsigned)__kmp_affinity_compact <= depth);
3969 KMP_DEBUG_ASSERT(__kmp_affinity_compact >= 0);
3970 for (i = 0; i < (unsigned)__kmp_affinity_compact; i++) {
3971 int j = depth - i - 1;
3972 if (aa->childNums[j] < bb->childNums[j])
3974 if (aa->childNums[j] > bb->childNums[j])
3977 for (; i < depth; i++) {
3978 int j = i - __kmp_affinity_compact;
3979 if (aa->childNums[j] < bb->childNums[j])
3981 if (aa->childNums[j] > bb->childNums[j])
3987 static void __kmp_aux_affinity_initialize(
void) {
3988 if (__kmp_affinity_masks != NULL) {
3989 KMP_ASSERT(__kmp_affin_fullMask != NULL);
3997 if (__kmp_affin_fullMask == NULL) {
3998 KMP_CPU_ALLOC(__kmp_affin_fullMask);
4000 if (KMP_AFFINITY_CAPABLE()) {
4001 if (__kmp_affinity_respect_mask) {
4002 __kmp_get_system_affinity(__kmp_affin_fullMask, TRUE);
4006 __kmp_avail_proc = 0;
4007 KMP_CPU_SET_ITERATE(i, __kmp_affin_fullMask) {
4008 if (!KMP_CPU_ISSET(i, __kmp_affin_fullMask)) {
4013 if (__kmp_avail_proc > __kmp_xproc) {
4014 if (__kmp_affinity_verbose ||
4015 (__kmp_affinity_warnings &&
4016 (__kmp_affinity_type != affinity_none))) {
4017 KMP_WARNING(ErrorInitializeAffinity);
4019 __kmp_affinity_type = affinity_none;
4020 KMP_AFFINITY_DISABLE();
4024 __kmp_affinity_entire_machine_mask(__kmp_affin_fullMask);
4025 __kmp_avail_proc = __kmp_xproc;
4029 if (__kmp_affinity_gran == affinity_gran_tile &&
4031 __kmp_affinity_dispatch->get_api_type() == KMPAffinity::NATIVE_OS) {
4032 KMP_WARNING(AffTilesNoHWLOC,
"KMP_AFFINITY");
4033 __kmp_affinity_gran = affinity_gran_package;
4037 kmp_i18n_id_t msg_id = kmp_i18n_null;
4041 if ((__kmp_cpuinfo_file != NULL) &&
4042 (__kmp_affinity_top_method == affinity_top_method_all)) {
4043 __kmp_affinity_top_method = affinity_top_method_cpuinfo;
4046 if (__kmp_affinity_top_method == affinity_top_method_all) {
4050 const char *file_name = NULL;
4054 __kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC) {
4055 if (__kmp_affinity_verbose) {
4056 KMP_INFORM(AffUsingHwloc,
"KMP_AFFINITY");
4058 if (!__kmp_hwloc_error) {
4059 depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
4062 }
else if (depth < 0 && __kmp_affinity_verbose) {
4063 KMP_INFORM(AffIgnoringHwloc,
"KMP_AFFINITY");
4065 }
else if (__kmp_affinity_verbose) {
4066 KMP_INFORM(AffIgnoringHwloc,
"KMP_AFFINITY");
4071 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4074 if (__kmp_affinity_verbose) {
4075 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY", KMP_I18N_STR(Decodingx2APIC));
4079 depth = __kmp_affinity_create_x2apicid_map(&address2os, &msg_id);
4085 if (__kmp_affinity_verbose) {
4086 if (msg_id != kmp_i18n_null) {
4087 KMP_INFORM(AffInfoStrStr,
"KMP_AFFINITY",
4088 __kmp_i18n_catgets(msg_id),
4089 KMP_I18N_STR(DecodingLegacyAPIC));
4091 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY",
4092 KMP_I18N_STR(DecodingLegacyAPIC));
4097 depth = __kmp_affinity_create_apicid_map(&address2os, &msg_id);
4109 if (__kmp_affinity_verbose) {
4110 if (msg_id != kmp_i18n_null) {
4111 KMP_INFORM(AffStrParseFilename,
"KMP_AFFINITY",
4112 __kmp_i18n_catgets(msg_id),
"/proc/cpuinfo");
4114 KMP_INFORM(AffParseFilename,
"KMP_AFFINITY",
"/proc/cpuinfo");
4118 FILE *f = fopen(
"/proc/cpuinfo",
"r");
4120 msg_id = kmp_i18n_str_CantOpenCpuinfo;
4122 file_name =
"/proc/cpuinfo";
4124 __kmp_affinity_create_cpuinfo_map(&address2os, &line, &msg_id, f);
4134 #if KMP_GROUP_AFFINITY 4136 if ((depth < 0) && (__kmp_num_proc_groups > 1)) {
4137 if (__kmp_affinity_verbose) {
4138 KMP_INFORM(AffWindowsProcGroupMap,
"KMP_AFFINITY");
4141 depth = __kmp_affinity_create_proc_group_map(&address2os, &msg_id);
4142 KMP_ASSERT(depth != 0);
4148 if (__kmp_affinity_verbose && (msg_id != kmp_i18n_null)) {
4149 if (file_name == NULL) {
4150 KMP_INFORM(UsingFlatOS, __kmp_i18n_catgets(msg_id));
4151 }
else if (line == 0) {
4152 KMP_INFORM(UsingFlatOSFile, file_name, __kmp_i18n_catgets(msg_id));
4154 KMP_INFORM(UsingFlatOSFileLine, file_name, line,
4155 __kmp_i18n_catgets(msg_id));
4161 depth = __kmp_affinity_create_flat_map(&address2os, &msg_id);
4165 KMP_ASSERT(depth > 0);
4166 KMP_ASSERT(address2os != NULL);
4171 else if (__kmp_affinity_top_method == affinity_top_method_hwloc) {
4172 KMP_ASSERT(__kmp_affinity_dispatch->get_api_type() == KMPAffinity::HWLOC);
4173 if (__kmp_affinity_verbose) {
4174 KMP_INFORM(AffUsingHwloc,
"KMP_AFFINITY");
4176 depth = __kmp_affinity_create_hwloc_map(&address2os, &msg_id);
4181 #endif // KMP_USE_HWLOC 4187 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 4189 else if (__kmp_affinity_top_method == affinity_top_method_x2apicid) {
4190 if (__kmp_affinity_verbose) {
4191 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY", KMP_I18N_STR(Decodingx2APIC));
4194 depth = __kmp_affinity_create_x2apicid_map(&address2os, &msg_id);
4199 KMP_ASSERT(msg_id != kmp_i18n_null);
4200 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4202 }
else if (__kmp_affinity_top_method == affinity_top_method_apicid) {
4203 if (__kmp_affinity_verbose) {
4204 KMP_INFORM(AffInfoStr,
"KMP_AFFINITY", KMP_I18N_STR(DecodingLegacyAPIC));
4207 depth = __kmp_affinity_create_apicid_map(&address2os, &msg_id);
4212 KMP_ASSERT(msg_id != kmp_i18n_null);
4213 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4219 else if (__kmp_affinity_top_method == affinity_top_method_cpuinfo) {
4220 const char *filename;
4221 if (__kmp_cpuinfo_file != NULL) {
4222 filename = __kmp_cpuinfo_file;
4224 filename =
"/proc/cpuinfo";
4227 if (__kmp_affinity_verbose) {
4228 KMP_INFORM(AffParseFilename,
"KMP_AFFINITY", filename);
4231 FILE *f = fopen(filename,
"r");
4234 if (__kmp_cpuinfo_file != NULL) {
4235 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4236 KMP_HNT(NameComesFrom_CPUINFO_FILE), __kmp_msg_null);
4238 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),
4243 depth = __kmp_affinity_create_cpuinfo_map(&address2os, &line, &msg_id, f);
4246 KMP_ASSERT(msg_id != kmp_i18n_null);
4248 KMP_FATAL(FileLineMsgExiting, filename, line,
4249 __kmp_i18n_catgets(msg_id));
4251 KMP_FATAL(FileMsgExiting, filename, __kmp_i18n_catgets(msg_id));
4254 if (__kmp_affinity_type == affinity_none) {
4255 KMP_ASSERT(depth == 0);
4260 #if KMP_GROUP_AFFINITY 4262 else if (__kmp_affinity_top_method == affinity_top_method_group) {
4263 if (__kmp_affinity_verbose) {
4264 KMP_INFORM(AffWindowsProcGroupMap,
"KMP_AFFINITY");
4267 depth = __kmp_affinity_create_proc_group_map(&address2os, &msg_id);
4268 KMP_ASSERT(depth != 0);
4270 KMP_ASSERT(msg_id != kmp_i18n_null);
4271 KMP_FATAL(MsgExiting, __kmp_i18n_catgets(msg_id));
4277 else if (__kmp_affinity_top_method == affinity_top_method_flat) {
4278 if (__kmp_affinity_verbose) {
4279 KMP_INFORM(AffUsingFlatOS,
"KMP_AFFINITY");
4282 depth = __kmp_affinity_create_flat_map(&address2os, &msg_id);
4287 KMP_ASSERT(depth > 0);
4288 KMP_ASSERT(address2os != NULL);
4291 if (address2os == NULL) {
4292 if (KMP_AFFINITY_CAPABLE() &&
4293 (__kmp_affinity_verbose ||
4294 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none)))) {
4295 KMP_WARNING(ErrorInitializeAffinity);
4297 __kmp_affinity_type = affinity_none;
4298 KMP_AFFINITY_DISABLE();
4302 if (__kmp_affinity_gran == affinity_gran_tile
4304 && __kmp_tile_depth == 0
4308 KMP_WARNING(AffTilesNoTiles,
"KMP_AFFINITY");
4311 __kmp_apply_thread_places(&address2os, depth);
4316 kmp_affin_mask_t *osId2Mask =
4317 __kmp_create_masks(&maxIndex, &numUnique, address2os, __kmp_avail_proc);
4318 if (__kmp_affinity_gran_levels == 0) {
4319 KMP_DEBUG_ASSERT((
int)numUnique == __kmp_avail_proc);
4325 __kmp_affinity_assign_child_nums(address2os, __kmp_avail_proc);
4327 switch (__kmp_affinity_type) {
4329 case affinity_explicit:
4330 KMP_DEBUG_ASSERT(__kmp_affinity_proclist != NULL);
4332 if (__kmp_nested_proc_bind.bind_types[0] == proc_bind_intel)
4335 __kmp_affinity_process_proclist(
4336 &__kmp_affinity_masks, &__kmp_affinity_num_masks,
4337 __kmp_affinity_proclist, osId2Mask, maxIndex);
4341 __kmp_affinity_process_placelist(
4342 &__kmp_affinity_masks, &__kmp_affinity_num_masks,
4343 __kmp_affinity_proclist, osId2Mask, maxIndex);
4346 if (__kmp_affinity_num_masks == 0) {
4347 if (__kmp_affinity_verbose ||
4348 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none))) {
4349 KMP_WARNING(AffNoValidProcID);
4351 __kmp_affinity_type = affinity_none;
4361 case affinity_logical:
4362 __kmp_affinity_compact = 0;
4363 if (__kmp_affinity_offset) {
4364 __kmp_affinity_offset =
4365 __kmp_nThreadsPerCore * __kmp_affinity_offset % __kmp_avail_proc;
4369 case affinity_physical:
4370 if (__kmp_nThreadsPerCore > 1) {
4371 __kmp_affinity_compact = 1;
4372 if (__kmp_affinity_compact >= depth) {
4373 __kmp_affinity_compact = 0;
4376 __kmp_affinity_compact = 0;
4378 if (__kmp_affinity_offset) {
4379 __kmp_affinity_offset =
4380 __kmp_nThreadsPerCore * __kmp_affinity_offset % __kmp_avail_proc;
4384 case affinity_scatter:
4385 if (__kmp_affinity_compact >= depth) {
4386 __kmp_affinity_compact = 0;
4388 __kmp_affinity_compact = depth - 1 - __kmp_affinity_compact;
4392 case affinity_compact:
4393 if (__kmp_affinity_compact >= depth) {
4394 __kmp_affinity_compact = depth - 1;
4398 case affinity_balanced:
4400 if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
4401 KMP_WARNING(AffBalancedNotAvail,
"KMP_AFFINITY");
4403 __kmp_affinity_type = affinity_none;
4405 }
else if (__kmp_affinity_uniform_topology()) {
4410 __kmp_aff_depth = depth;
4412 int core_level = __kmp_affinity_find_core_level(
4413 address2os, __kmp_avail_proc, depth - 1);
4414 int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
4415 depth - 1, core_level);
4416 int maxprocpercore = __kmp_affinity_max_proc_per_core(
4417 address2os, __kmp_avail_proc, depth - 1, core_level);
4419 int nproc = ncores * maxprocpercore;
4420 if ((nproc < 2) || (nproc < __kmp_avail_proc)) {
4421 if (__kmp_affinity_verbose || __kmp_affinity_warnings) {
4422 KMP_WARNING(AffBalancedNotAvail,
"KMP_AFFINITY");
4424 __kmp_affinity_type = affinity_none;
4428 procarr = (
int *)__kmp_allocate(
sizeof(
int) * nproc);
4429 for (
int i = 0; i < nproc; i++) {
4435 for (
int i = 0; i < __kmp_avail_proc; i++) {
4436 int proc = address2os[i].second;
4438 __kmp_affinity_find_core(address2os, i, depth - 1, core_level);
4440 if (core == lastcore) {
4447 procarr[core * maxprocpercore + inlastcore] = proc;
4455 if (__kmp_affinity_dups) {
4456 __kmp_affinity_num_masks = __kmp_avail_proc;
4458 __kmp_affinity_num_masks = numUnique;
4462 if ((__kmp_nested_proc_bind.bind_types[0] != proc_bind_intel) &&
4463 (__kmp_affinity_num_places > 0) &&
4464 ((
unsigned)__kmp_affinity_num_places < __kmp_affinity_num_masks)) {
4465 __kmp_affinity_num_masks = __kmp_affinity_num_places;
4469 KMP_CPU_ALLOC_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4473 qsort(address2os, __kmp_avail_proc,
sizeof(*address2os),
4474 __kmp_affinity_cmp_Address_child_num);
4478 for (i = 0, j = 0; i < __kmp_avail_proc; i++) {
4479 if ((!__kmp_affinity_dups) && (!address2os[i].first.leader)) {
4482 unsigned osId = address2os[i].second;
4483 kmp_affin_mask_t *src = KMP_CPU_INDEX(osId2Mask, osId);
4484 kmp_affin_mask_t *dest = KMP_CPU_INDEX(__kmp_affinity_masks, j);
4485 KMP_ASSERT(KMP_CPU_ISSET(osId, src));
4486 KMP_CPU_COPY(dest, src);
4487 if (++j >= __kmp_affinity_num_masks) {
4491 KMP_DEBUG_ASSERT(j == __kmp_affinity_num_masks);
4496 KMP_ASSERT2(0,
"Unexpected affinity setting");
4499 KMP_CPU_FREE_ARRAY(osId2Mask, maxIndex + 1);
4500 machine_hierarchy.init(address2os, __kmp_avail_proc);
4502 #undef KMP_EXIT_AFF_NONE 4504 void __kmp_affinity_initialize(
void) {
4513 int disabled = (__kmp_affinity_type == affinity_disabled);
4514 if (!KMP_AFFINITY_CAPABLE()) {
4515 KMP_ASSERT(disabled);
4518 __kmp_affinity_type = affinity_none;
4520 __kmp_aux_affinity_initialize();
4522 __kmp_affinity_type = affinity_disabled;
4526 void __kmp_affinity_uninitialize(
void) {
4527 if (__kmp_affinity_masks != NULL) {
4528 KMP_CPU_FREE_ARRAY(__kmp_affinity_masks, __kmp_affinity_num_masks);
4529 __kmp_affinity_masks = NULL;
4531 if (__kmp_affin_fullMask != NULL) {
4532 KMP_CPU_FREE(__kmp_affin_fullMask);
4533 __kmp_affin_fullMask = NULL;
4535 __kmp_affinity_num_masks = 0;
4536 __kmp_affinity_type = affinity_default;
4538 __kmp_affinity_num_places = 0;
4540 if (__kmp_affinity_proclist != NULL) {
4541 __kmp_free(__kmp_affinity_proclist);
4542 __kmp_affinity_proclist = NULL;
4544 if (address2os != NULL) {
4545 __kmp_free(address2os);
4548 if (procarr != NULL) {
4549 __kmp_free(procarr);
4553 if (__kmp_hwloc_topology != NULL) {
4554 hwloc_topology_destroy(__kmp_hwloc_topology);
4555 __kmp_hwloc_topology = NULL;
4558 KMPAffinity::destroy_api();
4561 void __kmp_affinity_set_init_mask(
int gtid,
int isa_root) {
4562 if (!KMP_AFFINITY_CAPABLE()) {
4566 kmp_info_t *th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[gtid]);
4567 if (th->th.th_affin_mask == NULL) {
4568 KMP_CPU_ALLOC(th->th.th_affin_mask);
4570 KMP_CPU_ZERO(th->th.th_affin_mask);
4577 kmp_affin_mask_t *mask;
4581 if (__kmp_nested_proc_bind.bind_types[0] == proc_bind_intel)
4584 if ((__kmp_affinity_type == affinity_none) ||
4585 (__kmp_affinity_type == affinity_balanced)) {
4586 #if KMP_GROUP_AFFINITY 4587 if (__kmp_num_proc_groups > 1) {
4591 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4593 mask = __kmp_affin_fullMask;
4595 KMP_DEBUG_ASSERT(__kmp_affinity_num_masks > 0);
4596 i = (gtid + __kmp_affinity_offset) % __kmp_affinity_num_masks;
4597 mask = KMP_CPU_INDEX(__kmp_affinity_masks, i);
4603 (__kmp_nested_proc_bind.bind_types[0] == proc_bind_false)) {
4604 #if KMP_GROUP_AFFINITY 4605 if (__kmp_num_proc_groups > 1) {
4609 KMP_ASSERT(__kmp_affin_fullMask != NULL);
4611 mask = __kmp_affin_fullMask;
4615 KMP_DEBUG_ASSERT(__kmp_affinity_num_masks > 0);
4616 i = (gtid + __kmp_affinity_offset) % __kmp_affinity_num_masks;
4617 mask = KMP_CPU_INDEX(__kmp_affinity_masks, i);
4623 th->th.th_current_place = i;
4625 th->th.th_new_place = i;
4626 th->th.th_first_place = 0;
4627 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4630 if (i == KMP_PLACE_ALL) {
4631 KA_TRACE(100, (
"__kmp_affinity_set_init_mask: binding T#%d to all places\n",
4634 KA_TRACE(100, (
"__kmp_affinity_set_init_mask: binding T#%d to place %d\n",
4641 (
"__kmp_affinity_set_init_mask: binding T#%d to __kmp_affin_fullMask\n",
4644 KA_TRACE(100, (
"__kmp_affinity_set_init_mask: binding T#%d to mask %d\n",
4649 KMP_CPU_COPY(th->th.th_affin_mask, mask);
4651 if (__kmp_affinity_verbose
4653 && (__kmp_affinity_type == affinity_none || i != KMP_PLACE_ALL)) {
4654 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4655 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4656 th->th.th_affin_mask);
4657 KMP_INFORM(BoundToOSProcSet,
"KMP_AFFINITY", (kmp_int32)getpid(),
4658 __kmp_gettid(), gtid, buf);
4665 if (__kmp_affinity_type == affinity_none) {
4666 __kmp_set_system_affinity(th->th.th_affin_mask, FALSE);
4669 __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
4674 void __kmp_affinity_set_place(
int gtid) {
4677 if (!KMP_AFFINITY_CAPABLE()) {
4681 kmp_info_t *th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[gtid]);
4683 KA_TRACE(100, (
"__kmp_affinity_set_place: binding T#%d to place %d (current " 4685 gtid, th->th.th_new_place, th->th.th_current_place));
4688 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4689 KMP_ASSERT(th->th.th_new_place >= 0);
4690 KMP_ASSERT((
unsigned)th->th.th_new_place <= __kmp_affinity_num_masks);
4691 if (th->th.th_first_place <= th->th.th_last_place) {
4692 KMP_ASSERT((th->th.th_new_place >= th->th.th_first_place) &&
4693 (th->th.th_new_place <= th->th.th_last_place));
4695 KMP_ASSERT((th->th.th_new_place <= th->th.th_first_place) ||
4696 (th->th.th_new_place >= th->th.th_last_place));
4701 kmp_affin_mask_t *mask =
4702 KMP_CPU_INDEX(__kmp_affinity_masks, th->th.th_new_place);
4703 KMP_CPU_COPY(th->th.th_affin_mask, mask);
4704 th->th.th_current_place = th->th.th_new_place;
4706 if (__kmp_affinity_verbose) {
4707 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4708 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4709 th->th.th_affin_mask);
4710 KMP_INFORM(BoundToOSProcSet,
"OMP_PROC_BIND", (kmp_int32)getpid(),
4711 __kmp_gettid(), gtid, buf);
4713 __kmp_set_system_affinity(th->th.th_affin_mask, TRUE);
4718 int __kmp_aux_set_affinity(
void **mask) {
4723 if (!KMP_AFFINITY_CAPABLE()) {
4727 gtid = __kmp_entry_gtid();
4729 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4730 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4731 (kmp_affin_mask_t *)(*mask));
4733 "kmp_set_affinity: setting affinity mask for thread %d = %s\n", gtid,
4737 if (__kmp_env_consistency_check) {
4738 if ((mask == NULL) || (*mask == NULL)) {
4739 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4744 KMP_CPU_SET_ITERATE(proc, ((kmp_affin_mask_t *)(*mask))) {
4745 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4746 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4748 if (!KMP_CPU_ISSET(proc, (kmp_affin_mask_t *)(*mask))) {
4753 if (num_procs == 0) {
4754 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4757 #if KMP_GROUP_AFFINITY 4758 if (__kmp_get_proc_group((kmp_affin_mask_t *)(*mask)) < 0) {
4759 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity");
4765 th = __kmp_threads[gtid];
4766 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4767 retval = __kmp_set_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
4769 KMP_CPU_COPY(th->th.th_affin_mask, (kmp_affin_mask_t *)(*mask));
4773 th->th.th_current_place = KMP_PLACE_UNDEFINED;
4774 th->th.th_new_place = KMP_PLACE_UNDEFINED;
4775 th->th.th_first_place = 0;
4776 th->th.th_last_place = __kmp_affinity_num_masks - 1;
4779 th->th.th_current_task->td_icvs.proc_bind = proc_bind_false;
4785 int __kmp_aux_get_affinity(
void **mask) {
4790 if (!KMP_AFFINITY_CAPABLE()) {
4794 gtid = __kmp_entry_gtid();
4795 th = __kmp_threads[gtid];
4796 KMP_DEBUG_ASSERT(th->th.th_affin_mask != NULL);
4799 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4800 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4801 th->th.th_affin_mask);
4802 __kmp_printf(
"kmp_get_affinity: stored affinity mask for thread %d = %s\n",
4806 if (__kmp_env_consistency_check) {
4807 if ((mask == NULL) || (*mask == NULL)) {
4808 KMP_FATAL(AffinityInvalidMask,
"kmp_get_affinity");
4814 retval = __kmp_get_system_affinity((kmp_affin_mask_t *)(*mask), FALSE);
4816 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4817 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4818 (kmp_affin_mask_t *)(*mask));
4819 __kmp_printf(
"kmp_get_affinity: system affinity mask for thread %d = %s\n",
4826 KMP_CPU_COPY((kmp_affin_mask_t *)(*mask), th->th.th_affin_mask);
4832 int __kmp_aux_get_affinity_max_proc() {
4833 if (!KMP_AFFINITY_CAPABLE()) {
4836 #if KMP_GROUP_AFFINITY 4837 if (__kmp_num_proc_groups > 1) {
4838 return (
int)(__kmp_num_proc_groups *
sizeof(DWORD_PTR) * CHAR_BIT);
4844 int __kmp_aux_set_affinity_mask_proc(
int proc,
void **mask) {
4847 if (!KMP_AFFINITY_CAPABLE()) {
4852 int gtid = __kmp_entry_gtid();
4853 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4854 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4855 (kmp_affin_mask_t *)(*mask));
4856 __kmp_debug_printf(
"kmp_set_affinity_mask_proc: setting proc %d in " 4857 "affinity mask for thread %d = %s\n",
4861 if (__kmp_env_consistency_check) {
4862 if ((mask == NULL) || (*mask == NULL)) {
4863 KMP_FATAL(AffinityInvalidMask,
"kmp_set_affinity_mask_proc");
4867 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
4870 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4874 KMP_CPU_SET(proc, (kmp_affin_mask_t *)(*mask));
4878 int __kmp_aux_unset_affinity_mask_proc(
int proc,
void **mask) {
4881 if (!KMP_AFFINITY_CAPABLE()) {
4886 int gtid = __kmp_entry_gtid();
4887 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4888 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4889 (kmp_affin_mask_t *)(*mask));
4890 __kmp_debug_printf(
"kmp_unset_affinity_mask_proc: unsetting proc %d in " 4891 "affinity mask for thread %d = %s\n",
4895 if (__kmp_env_consistency_check) {
4896 if ((mask == NULL) || (*mask == NULL)) {
4897 KMP_FATAL(AffinityInvalidMask,
"kmp_unset_affinity_mask_proc");
4901 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
4904 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4908 KMP_CPU_CLR(proc, (kmp_affin_mask_t *)(*mask));
4912 int __kmp_aux_get_affinity_mask_proc(
int proc,
void **mask) {
4915 if (!KMP_AFFINITY_CAPABLE()) {
4920 int gtid = __kmp_entry_gtid();
4921 char buf[KMP_AFFIN_MASK_PRINT_LEN];
4922 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
4923 (kmp_affin_mask_t *)(*mask));
4924 __kmp_debug_printf(
"kmp_get_affinity_mask_proc: getting proc %d in " 4925 "affinity mask for thread %d = %s\n",
4929 if (__kmp_env_consistency_check) {
4930 if ((mask == NULL) || (*mask == NULL)) {
4931 KMP_FATAL(AffinityInvalidMask,
"kmp_get_affinity_mask_proc");
4935 if ((proc < 0) || (proc >= __kmp_aux_get_affinity_max_proc())) {
4938 if (!KMP_CPU_ISSET(proc, __kmp_affin_fullMask)) {
4942 return KMP_CPU_ISSET(proc, (kmp_affin_mask_t *)(*mask));
4946 void __kmp_balanced_affinity(
int tid,
int nthreads) {
4947 bool fine_gran =
true;
4949 switch (__kmp_affinity_gran) {
4950 case affinity_gran_fine:
4951 case affinity_gran_thread:
4953 case affinity_gran_core:
4954 if (__kmp_nThreadsPerCore > 1) {
4958 case affinity_gran_package:
4959 if (nCoresPerPkg > 1) {
4967 if (__kmp_affinity_uniform_topology()) {
4971 int __kmp_nth_per_core = __kmp_avail_proc / __kmp_ncores;
4973 int ncores = __kmp_ncores;
4974 if ((nPackages > 1) && (__kmp_nth_per_core <= 1)) {
4975 __kmp_nth_per_core = __kmp_avail_proc / nPackages;
4979 int chunk = nthreads / ncores;
4981 int big_cores = nthreads % ncores;
4983 int big_nth = (chunk + 1) * big_cores;
4984 if (tid < big_nth) {
4985 coreID = tid / (chunk + 1);
4986 threadID = (tid % (chunk + 1)) % __kmp_nth_per_core;
4988 coreID = (tid - big_cores) / chunk;
4989 threadID = ((tid - big_cores) % chunk) % __kmp_nth_per_core;
4992 KMP_DEBUG_ASSERT2(KMP_AFFINITY_CAPABLE(),
4993 "Illegal set affinity operation when not capable");
4995 kmp_affin_mask_t *mask;
4996 KMP_CPU_ALLOC_ON_STACK(mask);
5000 int osID = address2os[coreID * __kmp_nth_per_core + threadID].second;
5001 KMP_CPU_SET(osID, mask);
5003 for (
int i = 0; i < __kmp_nth_per_core; i++) {
5005 osID = address2os[coreID * __kmp_nth_per_core + i].second;
5006 KMP_CPU_SET(osID, mask);
5009 if (__kmp_affinity_verbose) {
5010 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5011 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, mask);
5012 KMP_INFORM(BoundToOSProcSet,
"KMP_AFFINITY", (kmp_int32)getpid(),
5013 __kmp_gettid(), tid, buf);
5015 __kmp_set_system_affinity(mask, TRUE);
5016 KMP_CPU_FREE_FROM_STACK(mask);
5019 kmp_affin_mask_t *mask;
5020 KMP_CPU_ALLOC_ON_STACK(mask);
5023 int core_level = __kmp_affinity_find_core_level(
5024 address2os, __kmp_avail_proc, __kmp_aff_depth - 1);
5025 int ncores = __kmp_affinity_compute_ncores(address2os, __kmp_avail_proc,
5026 __kmp_aff_depth - 1, core_level);
5027 int nth_per_core = __kmp_affinity_max_proc_per_core(
5028 address2os, __kmp_avail_proc, __kmp_aff_depth - 1, core_level);
5032 if (nthreads == __kmp_avail_proc) {
5034 int osID = address2os[tid].second;
5035 KMP_CPU_SET(osID, mask);
5037 int core = __kmp_affinity_find_core(address2os, tid,
5038 __kmp_aff_depth - 1, core_level);
5039 for (
int i = 0; i < __kmp_avail_proc; i++) {
5040 int osID = address2os[i].second;
5041 if (__kmp_affinity_find_core(address2os, i, __kmp_aff_depth - 1,
5042 core_level) == core) {
5043 KMP_CPU_SET(osID, mask);
5047 }
else if (nthreads <= ncores) {
5050 for (
int i = 0; i < ncores; i++) {
5053 for (
int j = 0; j < nth_per_core; j++) {
5054 if (procarr[i * nth_per_core + j] != -1) {
5061 for (
int j = 0; j < nth_per_core; j++) {
5062 int osID = procarr[i * nth_per_core + j];
5064 KMP_CPU_SET(osID, mask);
5080 int *nproc_at_core = (
int *)KMP_ALLOCA(
sizeof(
int) * ncores);
5082 int *ncores_with_x_procs =
5083 (
int *)KMP_ALLOCA(
sizeof(
int) * (nth_per_core + 1));
5085 int *ncores_with_x_to_max_procs =
5086 (
int *)KMP_ALLOCA(
sizeof(
int) * (nth_per_core + 1));
5088 for (
int i = 0; i <= nth_per_core; i++) {
5089 ncores_with_x_procs[i] = 0;
5090 ncores_with_x_to_max_procs[i] = 0;
5093 for (
int i = 0; i < ncores; i++) {
5095 for (
int j = 0; j < nth_per_core; j++) {
5096 if (procarr[i * nth_per_core + j] != -1) {
5100 nproc_at_core[i] = cnt;
5101 ncores_with_x_procs[cnt]++;
5104 for (
int i = 0; i <= nth_per_core; i++) {
5105 for (
int j = i; j <= nth_per_core; j++) {
5106 ncores_with_x_to_max_procs[i] += ncores_with_x_procs[j];
5111 int nproc = nth_per_core * ncores;
5113 int *newarr = (
int *)__kmp_allocate(
sizeof(
int) * nproc);
5114 for (
int i = 0; i < nproc; i++) {
5121 for (
int j = 1; j <= nth_per_core; j++) {
5122 int cnt = ncores_with_x_to_max_procs[j];
5123 for (
int i = 0; i < ncores; i++) {
5125 if (nproc_at_core[i] == 0) {
5128 for (
int k = 0; k < nth_per_core; k++) {
5129 if (procarr[i * nth_per_core + k] != -1) {
5130 if (newarr[i * nth_per_core + k] == 0) {
5131 newarr[i * nth_per_core + k] = 1;
5137 newarr[i * nth_per_core + k]++;
5145 if (cnt == 0 || nth == 0) {
5156 for (
int i = 0; i < nproc; i++) {
5160 int osID = procarr[i];
5161 KMP_CPU_SET(osID, mask);
5163 int coreID = i / nth_per_core;
5164 for (
int ii = 0; ii < nth_per_core; ii++) {
5165 int osID = procarr[coreID * nth_per_core + ii];
5167 KMP_CPU_SET(osID, mask);
5177 if (__kmp_affinity_verbose) {
5178 char buf[KMP_AFFIN_MASK_PRINT_LEN];
5179 __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, mask);
5180 KMP_INFORM(BoundToOSProcSet,
"KMP_AFFINITY", (kmp_int32)getpid(),
5181 __kmp_gettid(), tid, buf);
5183 __kmp_set_system_affinity(mask, TRUE);
5184 KMP_CPU_FREE_FROM_STACK(mask);
5202 kmp_set_thread_affinity_mask_initial()
5207 int gtid = __kmp_get_gtid();
5210 KA_TRACE(30, (
"kmp_set_thread_affinity_mask_initial: " 5211 "non-omp thread, returning\n"));
5214 if (!KMP_AFFINITY_CAPABLE() || !__kmp_init_middle) {
5215 KA_TRACE(30, (
"kmp_set_thread_affinity_mask_initial: " 5216 "affinity not initialized, returning\n"));
5219 KA_TRACE(30, (
"kmp_set_thread_affinity_mask_initial: " 5220 "set full mask for thread %d\n",
5222 KMP_DEBUG_ASSERT(__kmp_affin_fullMask != NULL);
5223 return __kmp_set_system_affinity(__kmp_affin_fullMask, FALSE);
5227 #endif // KMP_AFFINITY_SUPPORTED