23 #if _MSC_VER && !__INTEL_COMPILER 24 #pragma warning( push ) 25 #pragma warning( disable : 4100 ) 28 #if _MSC_VER && !__INTEL_COMPILER 29 #pragma warning( pop ) 36 #define assertion_hwloc_wrapper(command, ...) \ 37 __TBB_ASSERT_EX( (command(__VA_ARGS__)) >= 0, "Error occurred during call to hwloc API."); 78 if ( hwloc_topology_init( &topology ) == 0 ) {
80 if ( hwloc_topology_load( topology ) == 0 ) {
88 hwloc_topology_destroy(topology);
91 numa_indexes_list.push_back(-1);
92 default_concurrency_list.push_back(-1);
98 process_cpu_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_cpuset (topology));
99 process_node_affinity_mask = hwloc_bitmap_dup(hwloc_topology_get_complete_nodeset(topology));
101 process_cpu_affinity_mask = hwloc_bitmap_alloc();
102 process_node_affinity_mask = hwloc_bitmap_alloc();
105 hwloc_cpuset_to_nodeset(topology, process_cpu_affinity_mask, process_node_affinity_mask);
111 if (hwloc_bitmap_weight(process_node_affinity_mask) < 0) {
112 numa_nodes_count = 1;
113 numa_indexes_list.push_back(0);
114 default_concurrency_list.push_back(hwloc_bitmap_weight(process_cpu_affinity_mask));
116 affinity_masks_list.push_back(hwloc_bitmap_dup(process_cpu_affinity_mask));
122 numa_nodes_count = hwloc_bitmap_weight(process_node_affinity_mask);
123 __TBB_ASSERT(numa_nodes_count > 0,
"Any system must contain one or more NUMA nodes");
126 unsigned counter = 0;
128 int max_numa_index = -1;
129 numa_indexes_list.resize(numa_nodes_count);
130 hwloc_obj_t node_buffer;
131 hwloc_bitmap_foreach_begin(i, process_node_affinity_mask) {
132 node_buffer = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, i);
133 numa_indexes_list[counter] =
static_cast<int>(node_buffer->logical_index);
135 if ( numa_indexes_list[counter] > max_numa_index ) {
136 max_numa_index = numa_indexes_list[counter];
140 } hwloc_bitmap_foreach_end();
141 __TBB_ASSERT(max_numa_index >= 0,
"Maximal NUMA index must not be negative");
144 default_concurrency_list.resize(max_numa_index + 1);
145 affinity_masks_list.resize(max_numa_index + 1);
148 hwloc_bitmap_foreach_begin(i, process_node_affinity_mask) {
149 node_buffer = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, i);
150 index =
static_cast<int>(node_buffer->logical_index);
152 hwloc_cpuset_t& current_mask = affinity_masks_list[index];
153 current_mask = hwloc_bitmap_dup(node_buffer->cpuset);
155 hwloc_bitmap_and(current_mask, current_mask, process_cpu_affinity_mask);
156 __TBB_ASSERT(!hwloc_bitmap_iszero(current_mask),
"hwloc detected unavailable NUMA node");
157 default_concurrency_list[index] = hwloc_bitmap_weight(current_mask);
158 } hwloc_bitmap_foreach_end();
165 hwloc_bitmap_free(affinity_masks_list[numa_indexes_list[i]]);
167 hwloc_bitmap_free(process_node_affinity_mask);
168 hwloc_bitmap_free(process_cpu_affinity_mask);
172 hwloc_topology_destroy(topology);
178 static void fill(
int& nodes_count,
int*& indexes_list,
int*& concurrency_list ) {
181 indexes_list = &numa_indexes_list.front();
182 concurrency_list = &default_concurrency_list.front();
187 return hwloc_bitmap_dup(process_cpu_affinity_mask);
191 hwloc_bitmap_free(mask_to_free);
197 hwloc_bitmap_and(current_mask,current_mask, process_cpu_affinity_mask);
199 "Current affinity mask must intersects with process affinity mask");
207 __TBB_ASSERT((
int)affinity_masks_list.size() > node_index,
208 "Trying to get affinity mask for uninitialized NUMA node");
209 return affinity_masks_list[node_index];
232 for (affinity_masks_container::iterator it = affinity_backup.begin();
233 it != affinity_backup.end(); it++) {
239 for (affinity_masks_container::iterator it = affinity_backup.begin();
240 it != affinity_backup.end(); it++) {
247 "The slot number is greater than the number of slots in the arena");
249 "Trying to get access to uninitialized platform_topology");
258 "Trying to get access to uninitialized platform_topology");
270 , my_numa_node_id(numa_id)
271 , numa_handler(num_slots)
286 int& nodes_count,
int*& indexes_list,
int*& concurrency_list ) {
293 __TBB_ASSERT(binding_observer,
"Failure during NUMA binding observer allocation and construction");
294 binding_observer->
observe(
true);
295 return binding_observer;
299 __TBB_ASSERT(binding_observer,
"Trying to deallocate NULL pointer");
300 binding_observer->
observe(
false);
301 delete binding_observer;
309 #undef assertion_hwloc_wrapper static hwloc_nodeset_t process_node_affinity_mask
static void fill(int &nodes_count, int *&indexes_list, int *&concurrency_list)
static void set_new_affinity_mask(const_affinity_mask new_mask)
hwloc_const_cpuset_t const_affinity_mask
void observe(bool state=true)
Enable or disable observation.
numa_affinity_handler numa_handler
static std::vector< int > default_concurrency_list
static bool intergroup_binding_allowed(size_t groups_num)
numa_binding_observer(task_arena *ta, int numa_id, int num_slots)
hwloc_cpuset_t affinity_mask
static std::vector< int > numa_indexes_list
task_scheduler_observer * subscribe_arena(task_arena *ta, int numa_id, int num_slots)
void restore_previous_affinity_mask(unsigned slot_num)
static void free_affinity_mask(affinity_mask mask_to_free)
affinity_masks_container affinity_backup
static init_stages initialization_state
static hwloc_topology_t topology
void on_scheduler_exit(bool) __TBB_override
Exit notification.
void on_scheduler_entry(bool) __TBB_override
Entry notification.
static void store_current_affinity_mask(affinity_mask current_mask)
static hwloc_cpuset_t process_cpu_affinity_mask
void unsubscribe_arena(task_scheduler_observer *binding_observer)
void initialize_numa_topology(size_t groups_num, int &nodes_count, int *&indexes_list, int *&concurrency_list)
static int numa_nodes_count
void bind_thread_to_node(unsigned slot_num, unsigned numa_node_id)
numa_affinity_handler(size_t size)
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
static void initialize(size_t groups_num)
#define assertion_hwloc_wrapper(command,...)
std::vector< platform_topology::affinity_mask > affinity_masks_container
static bool is_topology_parsed()
int current_thread_index()
Returns the index, aka slot number, of the calling thread in its current arena.
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t size
static std::vector< hwloc_cpuset_t > affinity_masks_list
static affinity_mask allocate_process_affinity_mask()
static const_affinity_mask get_node_affinity_mask(int node_index)