36 #if __TBB_SURVIVE_THREAD_SWITCH 40 #define CILKLIB_NAME "cilkrts20.dll" 42 #define CILKLIB_NAME "libcilkrts.so" 54 static atomic<do_once_state> cilkrts_load_state;
56 bool initialize_cilk_interop() {
69 int status = theTLS.create(auto_terminate);
71 int status = theTLS.create();
74 handle_perror(status,
"TBB failed to initialize task scheduler TLS\n");
80 theRMLServerFactory.close();
84 runtime_warning(
"TBB is unloaded while tbb::task_scheduler_init object is alive?" );
86 int status = theTLS.destroy();
88 runtime_warning(
"failed to destroy task scheduler TLS: %s", strerror(status));
93 rml::tbb_server* server = NULL;
94 if( !UsePrivateRML ) {
95 ::rml::factory::status_type status = theRMLServerFactory.make_server( server, client );
96 if( status != ::rml::factory::st_success ) {
98 runtime_warning(
"rml::tbb_factory::make_server failed with status %x, falling back on private rml", status );
111 __TBB_ASSERT( (uintptr_t(s)&1) == 0,
"Bad pointer to the scheduler" );
117 theTLS.set( tls_value_of(s) );
121 return theTLS.get() == tls_value_of(s);
126 assume_scheduler( s );
127 #if __TBB_SURVIVE_THREAD_SWITCH 128 if( watch_stack_handler ) {
132 if( (*watch_stack_handler)(&s->my_cilk_unwatch_thunk, o) ) {
134 s->my_cilk_unwatch_thunk.routine = NULL;
138 s->my_cilk_state = generic_scheduler::cs_running;
147 __TBB_ASSERT( is_set(s),
"attempt to unregister a wrong scheduler instance" );
148 assume_scheduler(NULL);
149 #if __TBB_SURVIVE_THREAD_SWITCH 159 #if __TBB_SURVIVE_THREAD_SWITCH 166 __TBB_ASSERT( is_set(NULL),
"TLS contains a scheduler?" );
174 if ( uintptr_t v = theTLS.get() ) {
183 #if __TBB_TASK_GROUP_CONTEXT 184 s->
my_arena->my_default_ctx = s->default_context();
187 assume_scheduler( s );
196 num_threads = default_num_threads();
199 __TBB_ASSERT(s,
"Somehow a local scheduler creation for a master thread failed");
207 __TBB_ASSERT( is_set(s),
"Attempt to terminate non-local scheduler instance" );
210 __TBB_ASSERT( is_set(NULL),
"cleanup_master has not cleared its TLS slot" );
224 __TBB_ASSERT( is_set(NULL),
"cleanup_master has not cleared its TLS slot" );
236 #if __TBB_SURVIVE_THREAD_SWITCH 237 if( watch_stack_handler )
243 ::rml::factory::status_type res = theRMLServerFactory.open();
244 UsePrivateRML = res != ::rml::factory::st_success;
247 #if __TBB_SURVIVE_THREAD_SWITCH 252 void* current = local_scheduler_if_initialized();
254 uintptr_t thread_id = GetCurrentThreadId();
256 uintptr_t thread_id = uintptr_t(pthread_self());
261 __TBB_ASSERT( !current && s->my_cilk_state==generic_scheduler::cs_limbo ||
262 current==s && s->my_cilk_state==generic_scheduler::cs_running,
"invalid adoption" );
265 runtime_warning(
"redundant adoption of %p by thread %p\n", s, (
void*)thread_id );
266 s->my_cilk_state = generic_scheduler::cs_running;
268 assume_scheduler( s );
272 __TBB_ASSERT( current==s && s->my_cilk_state==generic_scheduler::cs_running,
"invalid orphaning" );
274 s->my_cilk_state = generic_scheduler::cs_limbo;
276 assume_scheduler(NULL);
280 __TBB_ASSERT( !current && s->my_cilk_state==generic_scheduler::cs_limbo ||
281 current==s && s->my_cilk_state==generic_scheduler::cs_running,
"invalid release" );
283 s->my_cilk_state = generic_scheduler::cs_freed;
285 s->my_cilk_unwatch_thunk.routine = NULL;
296 #if __TBB_NUMA_SUPPORT 298 #if __TBB_WEAK_SYMBOLS_PRESENT 299 #pragma weak initialize_numa_topology 300 #pragma weak subscribe_arena 301 #pragma weak unsubscribe_arena 305 size_t groups_num,
int& nodes_count,
int*& indexes_list,
int*& concurrency_list );
313 #if _WIN32 || _WIN64 || __linux__ 314 static void (*initialize_numa_topology_handler)(
315 size_t groups_num,
int& nodes_count,
int*& indexes_list,
int*& concurrency_list ) = NULL;
321 static void (*unsubscribe_arena_handler)(
324 #if _WIN32 || _WIN64 || __linux__ 333 #define DEBUG_SUFFIX "_debug" 339 #define TBBBIND_NAME "tbbbind" DEBUG_SUFFIX ".dll" 341 #define TBBBIND_NAME "libtbbbind" DEBUG_SUFFIX __TBB_STRING(.so.TBB_COMPATIBLE_INTERFACE_VERSION) 354 namespace numa_topology {
356 int numa_nodes_count = 0;
357 int* numa_indexes = NULL;
358 int* default_concurrency_list = NULL;
359 static tbb::atomic<do_once_state> numa_topology_init_state;
365 void initialization_impl() {
368 #if _WIN32 || _WIN64 || __linux__ 369 bool load_tbbbind =
true;
370 #if _WIN32 && !_WIN64 373 GetNativeSystemInfo(&si);
374 load_tbbbind = si.dwNumberOfProcessors <= 32;
377 if (load_tbbbind &&
dynamic_link(TBBBIND_NAME, TbbBindLinkTable, 3)) {
378 int number_of_groups = 1;
380 number_of_groups = NumberOfProcessorGroups();
382 initialize_numa_topology_handler(
383 number_of_groups, numa_nodes_count, numa_indexes, default_concurrency_list);
385 if (numa_nodes_count==1 && numa_indexes[0] >= 0) {
387 "default_concurrency() should be equal to governor::default_num_threads() on single" 388 "NUMA node systems.");
394 static int dummy_index = -1;
397 numa_nodes_count = 1;
398 numa_indexes = &dummy_index;
399 default_concurrency_list = &dummy_concurrency;
401 subscribe_arena_handler = dummy_subscribe_arena;
402 unsubscribe_arena_handler = dummy_unsubscribe_arena;
409 unsigned nodes_count() {
411 return numa_nodes_count;
414 void fill(
int* indexes_array ) {
416 for (
int i = 0; i < numa_nodes_count; i++ ) {
417 indexes_array[i] = numa_indexes[i];
421 int default_concurrency(
int node_id ) {
424 return default_concurrency_list[node_id];
432 int numa_id,
int num_slots ) {
434 return (numa_id >= 0 && numa_topology::nodes_count() > 1) ?
435 subscribe_arena_handler(ta, numa_id, num_slots) : NULL;
439 __TBB_ASSERT(observer != NULL,
"Trying to access observer via NULL pointer");
440 unsubscribe_arena_handler(observer);
454 initialize( number_of_threads, 0 );
458 #if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS 459 uintptr_t new_mode = thread_stack_size & propagation_mode_mask;
462 if( number_of_threads!=deferred ) {
465 "number_of_threads for task_scheduler_init must be automatic or positive" );
467 #if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS 469 uintptr_t &vt = s->default_context()->my_version_and_traits;
472 : new_mode & propagation_mode_captured ? vt & ~
task_group_context::exact_exception : vt;
482 __TBB_ASSERT_RELEASE( !thread_stack_size,
"deferred initialization ignores stack size setting" );
487 #if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS 488 uintptr_t prev_mode = (uintptr_t)my_scheduler & propagation_mode_exact;
489 my_scheduler = (
scheduler*)((uintptr_t)my_scheduler & ~(uintptr_t)propagation_mode_exact);
493 __TBB_ASSERT_RELEASE( s,
"task_scheduler_init::terminate without corresponding task_scheduler_init::initialize()");
494 #if __TBB_TASK_GROUP_CONTEXT && TBB_USE_EXCEPTIONS 496 uintptr_t &vt = s->default_context()->my_version_and_traits;
505 internal_terminate(
false);
508 #if __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE 509 bool task_scheduler_init::internal_blocking_terminate(
bool throwing ) {
510 bool ok = internal_terminate(
true );
511 #if TBB_USE_EXCEPTIONS 512 if( throwing && !ok )
519 #endif // __TBB_SUPPORTS_WORKERS_WAITING_IN_TERMINATE __cilk_tbb_pfn_stack_op routine
static bool initialization_done()
void __TBB_EXPORTED_FUNC handle_perror(int error_code, const char *aux_info)
Throws std::runtime_error with what() returning error_code description prefixed with aux_info...
bool master_outermost_level() const
True if the scheduler is on the outermost dispatch level in a master thread.
static rml::tbb_server * create_rml_server(rml::tbb_client &)
void const char const char int ITT_FORMAT __itt_group_sync x void const char ITT_FORMAT __itt_group_sync s void ITT_FORMAT __itt_group_sync p void ITT_FORMAT p void ITT_FORMAT p no args __itt_suppress_mode_t unsigned int void size_t ITT_FORMAT d void ITT_FORMAT p void ITT_FORMAT p __itt_model_site __itt_model_site_instance ITT_FORMAT p __itt_model_task __itt_model_task_instance ITT_FORMAT p void ITT_FORMAT p void ITT_FORMAT p void size_t ITT_FORMAT d void ITT_FORMAT p const wchar_t ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s const char ITT_FORMAT s no args void ITT_FORMAT p size_t ITT_FORMAT d no args const wchar_t const wchar_t ITT_FORMAT s __itt_heap_function void size_t int ITT_FORMAT d __itt_heap_function void ITT_FORMAT p __itt_heap_function void void size_t int ITT_FORMAT d no args no args unsigned int ITT_FORMAT u const __itt_domain __itt_id ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain __itt_id ITT_FORMAT p const __itt_domain __itt_id __itt_timestamp __itt_timestamp ITT_FORMAT lu const __itt_domain __itt_id __itt_id __itt_string_handle ITT_FORMAT p const __itt_domain ITT_FORMAT p const __itt_domain __itt_string_handle unsigned long long ITT_FORMAT lu const __itt_domain __itt_id __itt_string_handle __itt_metadata_type size_t void * data
static void auto_terminate(void *scheduler)
The internal routine to undo automatic initialization.
#define __TBB_ASSERT_RELEASE(predicate, message)
static void release_resources()
Destroy the thread-local storage key and deinitialize RML.
static generic_scheduler * create_master(arena *a)
Initialize a scheduler for a master thread.
static generic_scheduler * init_scheduler(int num_threads, stack_size_type stack_size, bool auto_init)
Processes scheduler initialization request (possibly nested) in a master thread.
task_scheduler_observer * subscribe_arena(task_arena *ta, int numa_id, int num_slots)
__cilk_tbb_pfn_unwatch_stacks routine
static bool terminate_scheduler(generic_scheduler *s, bool blocking)
Processes scheduler termination request (possibly nested) in a master thread.
const int DYNAMIC_LINK_GLOBAL
bool is_worker() const
True if running on a worker thread, false otherwise.
bool my_auto_initialized
True if *this was created by automatic TBB initialization.
static unsigned default_num_threads()
static const int automatic
Typedef for number of threads that is automatic.
static bool is_set(generic_scheduler *s)
Used to check validity of the local scheduler TLS contents.
arena * my_arena
The arena that I own (if master) or am servicing at the moment (if worker)
bool gcc_rethrow_exception_broken()
void dynamic_unlink_all()
void DoOneTimeInitializations()
Performs thread-safe lazy one-time general TBB initialization.
static generic_scheduler * init_scheduler_weak()
Automatic initialization of scheduler in a master thread with default settings without arena...
void PrintExtraVersionInfo(const char *category, const char *format,...)
Prints arbitrary extra TBB version information on stderr.
void const char const char int ITT_FORMAT __itt_group_sync s
static void initialize_rml_factory()
void attach_arena(arena *, size_t index, bool is_master)
void __TBB_EXPORTED_FUNC runtime_warning(const char *format,...)
Report a runtime warning.
void throw_exception(exception_id eid)
Versionless convenience wrapper for throw_exception_v4()
void unsubscribe_arena(task_scheduler_observer *binding_observer)
void PrintRMLVersionInfo(void *arg, const char *server_info)
A callback routine to print RML version information on stderr.
static void print_version_info()
void initialize_numa_topology(size_t groups_num, int &nodes_count, int *&indexes_list, int *&concurrency_list)
size_t my_arena_index
Index of the arena slot the scheduler occupies now, or occupied last time.
OPEN_INTERNAL_NAMESPACE bool dynamic_link(const char *, const dynamic_link_descriptor *, size_t, dynamic_link_handle *handle, int)
static void sign_off(generic_scheduler *s)
Unregister TBB scheduler instance from thread-local storage.
long my_ref_count
Reference count for scheduler.
static int __TBB_EXPORTED_FUNC default_num_threads()
Returns the number of threads TBB scheduler would create if initialized by default.
static void assume_scheduler(generic_scheduler *s)
Temporarily set TLS slot to the given scheduler.
static uintptr_t tls_value_of(generic_scheduler *s)
Computes the value of the TLS.
bool internal_terminate(bool blocking)
void suppress_unused_warning(const T1 &)
Utility template function to prevent "unused" warnings by various compilers.
static void one_time_init()
Used to form groups of tasks.
#define __TBB_ASSERT(predicate, comment)
No-op version of __TBB_ASSERT.
void atomic_do_once(const F &initializer, atomic< do_once_state > &state)
One-time initialization function.
void __TBB_EXPORTED_METHOD terminate()
Inverse of method initialize.
#define DLD(s, h)
The helper to construct dynamic_link_descriptor structure.
CILK_EXPORT __cilk_tbb_retcode __cilkrts_watch_stack(struct __cilk_tbb_unwatch_thunk *u, struct __cilk_tbb_stack_op_thunk o)
static void acquire_resources()
Create key for thread-local storage and initialize RML.
std::size_t stack_size_type
bool cleanup_master(bool blocking_terminate)
Perform necessary cleanup when a master thread stops using TBB.
static arena * create_arena(int num_slots, int num_reserved_slots, size_t stack_size)
Creates an arena object.
Work stealing task scheduler.
Association between a handler name and location of pointer to it.
static void sign_on(generic_scheduler *s)
Register TBB scheduler instance in thread-local storage.
void destroy_process_mask()
generic_scheduler * my_scheduler
Scheduler of the thread attached to the slot.
tbb_server * make_private_server(tbb_client &client)
Factory method called from task.cpp to create a private_server.
arena_slot * my_arena_slot
Pointer to the slot in the arena we own at the moment.
void __TBB_EXPORTED_METHOD initialize(int number_of_threads=automatic)
Ensure that scheduler exists for this thread.
bool cpu_has_speculation()
check for transaction support.