diff options
| author | arcadia-devtools <[email protected]> | 2022-03-01 22:49:23 +0300 |
|---|---|---|
| committer | arcadia-devtools <[email protected]> | 2022-03-01 22:49:23 +0300 |
| commit | f1db7e1d2a6f1e911c41352aecb7897b8cc48d74 (patch) | |
| tree | e551b29a4f1a5e46cb6f23b04e3192dcf5f9da32 /contrib/libs/cxxsupp/openmp/kmp_threadprivate.cpp | |
| parent | 8de79fac61fafe1e9e559da116135cca3f5846d1 (diff) | |
intermediate changes
ref:51d474bda1b99a2cf73ca7da0cd5398ef5683bf4
Diffstat (limited to 'contrib/libs/cxxsupp/openmp/kmp_threadprivate.cpp')
| -rw-r--r-- | contrib/libs/cxxsupp/openmp/kmp_threadprivate.cpp | 798 |
1 files changed, 0 insertions, 798 deletions
diff --git a/contrib/libs/cxxsupp/openmp/kmp_threadprivate.cpp b/contrib/libs/cxxsupp/openmp/kmp_threadprivate.cpp deleted file mode 100644 index b79ac7d6d2b..00000000000 --- a/contrib/libs/cxxsupp/openmp/kmp_threadprivate.cpp +++ /dev/null @@ -1,798 +0,0 @@ -/* - * kmp_threadprivate.cpp -- OpenMP threadprivate support library - */ - -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "kmp.h" -#include "kmp_i18n.h" -#include "kmp_itt.h" - -#define USE_CHECKS_COMMON - -#define KMP_INLINE_SUBR 1 - -void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr, - void *data_addr, size_t pc_size); -struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr, - void *data_addr, - size_t pc_size); - -struct shared_table __kmp_threadprivate_d_table; - -static -#ifdef KMP_INLINE_SUBR - __forceinline -#endif - struct private_common * - __kmp_threadprivate_find_task_common(struct common_table *tbl, int gtid, - void *pc_addr) - -{ - struct private_common *tn; - -#ifdef KMP_TASK_COMMON_DEBUG - KC_TRACE(10, ("__kmp_threadprivate_find_task_common: thread#%d, called with " - "address %p\n", - gtid, pc_addr)); - dump_list(); -#endif - - for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) { - if (tn->gbl_addr == pc_addr) { -#ifdef KMP_TASK_COMMON_DEBUG - KC_TRACE(10, ("__kmp_threadprivate_find_task_common: thread#%d, found " - "node %p on list\n", - gtid, pc_addr)); -#endif - return tn; - } - } - return 0; -} - -static -#ifdef KMP_INLINE_SUBR - __forceinline -#endif - struct shared_common * - __kmp_find_shared_task_common(struct shared_table *tbl, int gtid, - void *pc_addr) { - struct shared_common *tn; - - for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) { - if (tn->gbl_addr == pc_addr) { -#ifdef KMP_TASK_COMMON_DEBUG - KC_TRACE( - 10, - ("__kmp_find_shared_task_common: thread#%d, found node %p on list\n", - gtid, pc_addr)); -#endif - return tn; - } - } - return 0; -} - -// Create a template for the data initialized storage. Either the template is -// NULL indicating zero fill, or the template is a copy of the original data. -static struct private_data *__kmp_init_common_data(void *pc_addr, - size_t pc_size) { - struct private_data *d; - size_t i; - char *p; - - d = (struct private_data *)__kmp_allocate(sizeof(struct private_data)); - /* - d->data = 0; // AC: commented out because __kmp_allocate zeroes the - memory - d->next = 0; - */ - d->size = pc_size; - d->more = 1; - - p = (char *)pc_addr; - - for (i = pc_size; i > 0; --i) { - if (*p++ != '\0') { - d->data = __kmp_allocate(pc_size); - KMP_MEMCPY(d->data, pc_addr, pc_size); - break; - } - } - - return d; -} - -// Initialize the data area from the template. -static void __kmp_copy_common_data(void *pc_addr, struct private_data *d) { - char *addr = (char *)pc_addr; - - for (size_t offset = 0; d != 0; d = d->next) { - for (int i = d->more; i > 0; --i) { - if (d->data == 0) - memset(&addr[offset], '\0', d->size); - else - KMP_MEMCPY(&addr[offset], d->data, d->size); - offset += d->size; - } - } -} - -/* we are called from __kmp_serial_initialize() with __kmp_initz_lock held. */ -void __kmp_common_initialize(void) { - if (!TCR_4(__kmp_init_common)) { - int q; -#ifdef KMP_DEBUG - int gtid; -#endif - - __kmp_threadpriv_cache_list = NULL; - -#ifdef KMP_DEBUG - /* verify the uber masters were initialized */ - for (gtid = 0; gtid < __kmp_threads_capacity; gtid++) - if (__kmp_root[gtid]) { - KMP_DEBUG_ASSERT(__kmp_root[gtid]->r.r_uber_thread); - for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) - KMP_DEBUG_ASSERT( - !__kmp_root[gtid]->r.r_uber_thread->th.th_pri_common->data[q]); - /* __kmp_root[ gitd ]-> r.r_uber_thread -> - * th.th_pri_common -> data[ q ] = 0;*/ - } -#endif /* KMP_DEBUG */ - - for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) - __kmp_threadprivate_d_table.data[q] = 0; - - TCW_4(__kmp_init_common, TRUE); - } -} - -/* Call all destructors for threadprivate data belonging to all threads. - Currently unused! */ -void __kmp_common_destroy(void) { - if (TCR_4(__kmp_init_common)) { - int q; - - TCW_4(__kmp_init_common, FALSE); - - for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) { - int gtid; - struct private_common *tn; - struct shared_common *d_tn; - - /* C++ destructors need to be called once per thread before exiting. - Don't call destructors for primary thread though unless we used copy - constructor */ - - for (d_tn = __kmp_threadprivate_d_table.data[q]; d_tn; - d_tn = d_tn->next) { - if (d_tn->is_vec) { - if (d_tn->dt.dtorv != 0) { - for (gtid = 0; gtid < __kmp_all_nth; ++gtid) { - if (__kmp_threads[gtid]) { - if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) - : (!KMP_UBER_GTID(gtid))) { - tn = __kmp_threadprivate_find_task_common( - __kmp_threads[gtid]->th.th_pri_common, gtid, - d_tn->gbl_addr); - if (tn) { - (*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len); - } - } - } - } - if (d_tn->obj_init != 0) { - (*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len); - } - } - } else { - if (d_tn->dt.dtor != 0) { - for (gtid = 0; gtid < __kmp_all_nth; ++gtid) { - if (__kmp_threads[gtid]) { - if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) - : (!KMP_UBER_GTID(gtid))) { - tn = __kmp_threadprivate_find_task_common( - __kmp_threads[gtid]->th.th_pri_common, gtid, - d_tn->gbl_addr); - if (tn) { - (*d_tn->dt.dtor)(tn->par_addr); - } - } - } - } - if (d_tn->obj_init != 0) { - (*d_tn->dt.dtor)(d_tn->obj_init); - } - } - } - } - __kmp_threadprivate_d_table.data[q] = 0; - } - } -} - -/* Call all destructors for threadprivate data belonging to this thread */ -void __kmp_common_destroy_gtid(int gtid) { - struct private_common *tn; - struct shared_common *d_tn; - - if (!TCR_4(__kmp_init_gtid)) { - // This is possible when one of multiple roots initiates early library - // termination in a sequential region while other teams are active, and its - // child threads are about to end. - return; - } - - KC_TRACE(10, ("__kmp_common_destroy_gtid: T#%d called\n", gtid)); - if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) : (!KMP_UBER_GTID(gtid))) { - - if (TCR_4(__kmp_init_common)) { - - /* Cannot do this here since not all threads have destroyed their data */ - /* TCW_4(__kmp_init_common, FALSE); */ - - for (tn = __kmp_threads[gtid]->th.th_pri_head; tn; tn = tn->link) { - - d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid, - tn->gbl_addr); - if (d_tn == NULL) - continue; - if (d_tn->is_vec) { - if (d_tn->dt.dtorv != 0) { - (void)(*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len); - } - if (d_tn->obj_init != 0) { - (void)(*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len); - } - } else { - if (d_tn->dt.dtor != 0) { - (void)(*d_tn->dt.dtor)(tn->par_addr); - } - if (d_tn->obj_init != 0) { - (void)(*d_tn->dt.dtor)(d_tn->obj_init); - } - } - } - KC_TRACE(30, ("__kmp_common_destroy_gtid: T#%d threadprivate destructors " - "complete\n", - gtid)); - } - } -} - -#ifdef KMP_TASK_COMMON_DEBUG -static void dump_list(void) { - int p, q; - - for (p = 0; p < __kmp_all_nth; ++p) { - if (!__kmp_threads[p]) - continue; - for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) { - if (__kmp_threads[p]->th.th_pri_common->data[q]) { - struct private_common *tn; - - KC_TRACE(10, ("\tdump_list: gtid:%d addresses\n", p)); - - for (tn = __kmp_threads[p]->th.th_pri_common->data[q]; tn; - tn = tn->next) { - KC_TRACE(10, - ("\tdump_list: THREADPRIVATE: Serial %p -> Parallel %p\n", - tn->gbl_addr, tn->par_addr)); - } - } - } - } -} -#endif /* KMP_TASK_COMMON_DEBUG */ - -// NOTE: this routine is to be called only from the serial part of the program. -void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr, - void *data_addr, size_t pc_size) { - struct shared_common **lnk_tn, *d_tn; - KMP_DEBUG_ASSERT(__kmp_threads[gtid] && - __kmp_threads[gtid]->th.th_root->r.r_active == 0); - - d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid, - pc_addr); - - if (d_tn == 0) { - d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); - - d_tn->gbl_addr = pc_addr; - d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size); - /* - d_tn->obj_init = 0; // AC: commented out because __kmp_allocate - zeroes the memory - d_tn->ct.ctor = 0; - d_tn->cct.cctor = 0;; - d_tn->dt.dtor = 0; - d_tn->is_vec = FALSE; - d_tn->vec_len = 0L; - */ - d_tn->cmn_size = pc_size; - - __kmp_acquire_lock(&__kmp_global_lock, gtid); - - lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]); - - d_tn->next = *lnk_tn; - *lnk_tn = d_tn; - - __kmp_release_lock(&__kmp_global_lock, gtid); - } -} - -struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr, - void *data_addr, - size_t pc_size) { - struct private_common *tn, **tt; - struct shared_common *d_tn; - - /* +++++++++ START OF CRITICAL SECTION +++++++++ */ - __kmp_acquire_lock(&__kmp_global_lock, gtid); - - tn = (struct private_common *)__kmp_allocate(sizeof(struct private_common)); - - tn->gbl_addr = pc_addr; - - d_tn = __kmp_find_shared_task_common( - &__kmp_threadprivate_d_table, gtid, - pc_addr); /* Only the MASTER data table exists. */ - - if (d_tn != 0) { - /* This threadprivate variable has already been seen. */ - - if (d_tn->pod_init == 0 && d_tn->obj_init == 0) { - d_tn->cmn_size = pc_size; - - if (d_tn->is_vec) { - if (d_tn->ct.ctorv != 0) { - /* Construct from scratch so no prototype exists */ - d_tn->obj_init = 0; - } else if (d_tn->cct.cctorv != 0) { - /* Now data initialize the prototype since it was previously - * registered */ - d_tn->obj_init = (void *)__kmp_allocate(d_tn->cmn_size); - (void)(*d_tn->cct.cctorv)(d_tn->obj_init, pc_addr, d_tn->vec_len); - } else { - d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size); - } - } else { - if (d_tn->ct.ctor != 0) { - /* Construct from scratch so no prototype exists */ - d_tn->obj_init = 0; - } else if (d_tn->cct.cctor != 0) { - /* Now data initialize the prototype since it was previously - registered */ - d_tn->obj_init = (void *)__kmp_allocate(d_tn->cmn_size); - (void)(*d_tn->cct.cctor)(d_tn->obj_init, pc_addr); - } else { - d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size); - } - } - } - } else { - struct shared_common **lnk_tn; - - d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); - d_tn->gbl_addr = pc_addr; - d_tn->cmn_size = pc_size; - d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size); - /* - d_tn->obj_init = 0; // AC: commented out because __kmp_allocate - zeroes the memory - d_tn->ct.ctor = 0; - d_tn->cct.cctor = 0; - d_tn->dt.dtor = 0; - d_tn->is_vec = FALSE; - d_tn->vec_len = 0L; - */ - lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]); - - d_tn->next = *lnk_tn; - *lnk_tn = d_tn; - } - - tn->cmn_size = d_tn->cmn_size; - - if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid))) { - tn->par_addr = (void *)pc_addr; - } else { - tn->par_addr = (void *)__kmp_allocate(tn->cmn_size); - } - - __kmp_release_lock(&__kmp_global_lock, gtid); - /* +++++++++ END OF CRITICAL SECTION +++++++++ */ - -#ifdef USE_CHECKS_COMMON - if (pc_size > d_tn->cmn_size) { - KC_TRACE( - 10, ("__kmp_threadprivate_insert: THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC - " ,%" KMP_UINTPTR_SPEC ")\n", - pc_addr, pc_size, d_tn->cmn_size)); - KMP_FATAL(TPCommonBlocksInconsist); - } -#endif /* USE_CHECKS_COMMON */ - - tt = &(__kmp_threads[gtid]->th.th_pri_common->data[KMP_HASH(pc_addr)]); - -#ifdef KMP_TASK_COMMON_DEBUG - if (*tt != 0) { - KC_TRACE( - 10, - ("__kmp_threadprivate_insert: WARNING! thread#%d: collision on %p\n", - gtid, pc_addr)); - } -#endif - tn->next = *tt; - *tt = tn; - -#ifdef KMP_TASK_COMMON_DEBUG - KC_TRACE(10, - ("__kmp_threadprivate_insert: thread#%d, inserted node %p on list\n", - gtid, pc_addr)); - dump_list(); -#endif - - /* Link the node into a simple list */ - - tn->link = __kmp_threads[gtid]->th.th_pri_head; - __kmp_threads[gtid]->th.th_pri_head = tn; - - if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid))) - return tn; - - /* if C++ object with copy constructor, use it; - * else if C++ object with constructor, use it for the non-primary thread - copies only; - * else use pod_init and memcpy - * - * C++ constructors need to be called once for each non-primary thread on - * allocate - * C++ copy constructors need to be called once for each thread on allocate */ - - /* C++ object with constructors/destructors; don't call constructors for - primary thread though */ - if (d_tn->is_vec) { - if (d_tn->ct.ctorv != 0) { - (void)(*d_tn->ct.ctorv)(tn->par_addr, d_tn->vec_len); - } else if (d_tn->cct.cctorv != 0) { - (void)(*d_tn->cct.cctorv)(tn->par_addr, d_tn->obj_init, d_tn->vec_len); - } else if (tn->par_addr != tn->gbl_addr) { - __kmp_copy_common_data(tn->par_addr, d_tn->pod_init); - } - } else { - if (d_tn->ct.ctor != 0) { - (void)(*d_tn->ct.ctor)(tn->par_addr); - } else if (d_tn->cct.cctor != 0) { - (void)(*d_tn->cct.cctor)(tn->par_addr, d_tn->obj_init); - } else if (tn->par_addr != tn->gbl_addr) { - __kmp_copy_common_data(tn->par_addr, d_tn->pod_init); - } - } - /* !BUILD_OPENMP_C - if (tn->par_addr != tn->gbl_addr) - __kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); */ - - return tn; -} - -/* ------------------------------------------------------------------------ */ -/* We are currently parallel, and we know the thread id. */ -/* ------------------------------------------------------------------------ */ - -/*! - @ingroup THREADPRIVATE - - @param loc source location information - @param data pointer to data being privatized - @param ctor pointer to constructor function for data - @param cctor pointer to copy constructor function for data - @param dtor pointer to destructor function for data - - Register constructors and destructors for thread private data. - This function is called when executing in parallel, when we know the thread id. -*/ -void __kmpc_threadprivate_register(ident_t *loc, void *data, kmpc_ctor ctor, - kmpc_cctor cctor, kmpc_dtor dtor) { - struct shared_common *d_tn, **lnk_tn; - - KC_TRACE(10, ("__kmpc_threadprivate_register: called\n")); - -#ifdef USE_CHECKS_COMMON - /* copy constructor must be zero for current code gen (Nov 2002 - jph) */ - KMP_ASSERT(cctor == 0); -#endif /* USE_CHECKS_COMMON */ - - /* Only the global data table exists. */ - d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, -1, data); - - if (d_tn == 0) { - d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); - d_tn->gbl_addr = data; - - d_tn->ct.ctor = ctor; - d_tn->cct.cctor = cctor; - d_tn->dt.dtor = dtor; - /* - d_tn->is_vec = FALSE; // AC: commented out because __kmp_allocate - zeroes the memory - d_tn->vec_len = 0L; - d_tn->obj_init = 0; - d_tn->pod_init = 0; - */ - lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]); - - d_tn->next = *lnk_tn; - *lnk_tn = d_tn; - } -} - -void *__kmpc_threadprivate(ident_t *loc, kmp_int32 global_tid, void *data, - size_t size) { - void *ret; - struct private_common *tn; - - KC_TRACE(10, ("__kmpc_threadprivate: T#%d called\n", global_tid)); - -#ifdef USE_CHECKS_COMMON - if (!__kmp_init_serial) - KMP_FATAL(RTLNotInitialized); -#endif /* USE_CHECKS_COMMON */ - - if (!__kmp_threads[global_tid]->th.th_root->r.r_active && !__kmp_foreign_tp) { - /* The parallel address will NEVER overlap with the data_address */ - /* dkp: 3rd arg to kmp_threadprivate_insert_private_data() is the - * data_address; use data_address = data */ - - KC_TRACE(20, ("__kmpc_threadprivate: T#%d inserting private data\n", - global_tid)); - kmp_threadprivate_insert_private_data(global_tid, data, data, size); - - ret = data; - } else { - KC_TRACE( - 50, - ("__kmpc_threadprivate: T#%d try to find private data at address %p\n", - global_tid, data)); - tn = __kmp_threadprivate_find_task_common( - __kmp_threads[global_tid]->th.th_pri_common, global_tid, data); - - if (tn) { - KC_TRACE(20, ("__kmpc_threadprivate: T#%d found data\n", global_tid)); -#ifdef USE_CHECKS_COMMON - if ((size_t)size > tn->cmn_size) { - KC_TRACE(10, ("THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC - " ,%" KMP_UINTPTR_SPEC ")\n", - data, size, tn->cmn_size)); - KMP_FATAL(TPCommonBlocksInconsist); - } -#endif /* USE_CHECKS_COMMON */ - } else { - /* The parallel address will NEVER overlap with the data_address */ - /* dkp: 3rd arg to kmp_threadprivate_insert() is the data_address; use - * data_address = data */ - KC_TRACE(20, ("__kmpc_threadprivate: T#%d inserting data\n", global_tid)); - tn = kmp_threadprivate_insert(global_tid, data, data, size); - } - - ret = tn->par_addr; - } - KC_TRACE(10, ("__kmpc_threadprivate: T#%d exiting; return value = %p\n", - global_tid, ret)); - - return ret; -} - -static kmp_cached_addr_t *__kmp_find_cache(void *data) { - kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list; - while (ptr && ptr->data != data) - ptr = ptr->next; - return ptr; -} - -/*! - @ingroup THREADPRIVATE - @param loc source location information - @param global_tid global thread number - @param data pointer to data to privatize - @param size size of data to privatize - @param cache pointer to cache - @return pointer to private storage - - Allocate private storage for threadprivate data. -*/ -void * -__kmpc_threadprivate_cached(ident_t *loc, - kmp_int32 global_tid, // gtid. - void *data, // Pointer to original global variable. - size_t size, // Size of original global variable. - void ***cache) { - KC_TRACE(10, ("__kmpc_threadprivate_cached: T#%d called with cache: %p, " - "address: %p, size: %" KMP_SIZE_T_SPEC "\n", - global_tid, *cache, data, size)); - - if (TCR_PTR(*cache) == 0) { - __kmp_acquire_lock(&__kmp_global_lock, global_tid); - - if (TCR_PTR(*cache) == 0) { - __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); - // Compiler often passes in NULL cache, even if it's already been created - void **my_cache; - kmp_cached_addr_t *tp_cache_addr; - // Look for an existing cache - tp_cache_addr = __kmp_find_cache(data); - if (!tp_cache_addr) { // Cache was never created; do it now - __kmp_tp_cached = 1; - KMP_ITT_IGNORE(my_cache = (void **)__kmp_allocate( - sizeof(void *) * __kmp_tp_capacity + - sizeof(kmp_cached_addr_t));); - // No need to zero the allocated memory; __kmp_allocate does that. - KC_TRACE(50, ("__kmpc_threadprivate_cached: T#%d allocated cache at " - "address %p\n", - global_tid, my_cache)); - /* TODO: free all this memory in __kmp_common_destroy using - * __kmp_threadpriv_cache_list */ - /* Add address of mycache to linked list for cleanup later */ - tp_cache_addr = (kmp_cached_addr_t *)&my_cache[__kmp_tp_capacity]; - tp_cache_addr->addr = my_cache; - tp_cache_addr->data = data; - tp_cache_addr->compiler_cache = cache; - tp_cache_addr->next = __kmp_threadpriv_cache_list; - __kmp_threadpriv_cache_list = tp_cache_addr; - } else { // A cache was already created; use it - my_cache = tp_cache_addr->addr; - tp_cache_addr->compiler_cache = cache; - } - KMP_MB(); - - TCW_PTR(*cache, my_cache); - __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); - - KMP_MB(); - } - __kmp_release_lock(&__kmp_global_lock, global_tid); - } - - void *ret; - if ((ret = TCR_PTR((*cache)[global_tid])) == 0) { - ret = __kmpc_threadprivate(loc, global_tid, data, (size_t)size); - - TCW_PTR((*cache)[global_tid], ret); - } - KC_TRACE(10, - ("__kmpc_threadprivate_cached: T#%d exiting; return value = %p\n", - global_tid, ret)); - return ret; -} - -// This function should only be called when both __kmp_tp_cached_lock and -// kmp_forkjoin_lock are held. -void __kmp_threadprivate_resize_cache(int newCapacity) { - KC_TRACE(10, ("__kmp_threadprivate_resize_cache: called with size: %d\n", - newCapacity)); - - kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list; - - while (ptr) { - if (ptr->data) { // this location has an active cache; resize it - void **my_cache; - KMP_ITT_IGNORE(my_cache = - (void **)__kmp_allocate(sizeof(void *) * newCapacity + - sizeof(kmp_cached_addr_t));); - // No need to zero the allocated memory; __kmp_allocate does that. - KC_TRACE(50, ("__kmp_threadprivate_resize_cache: allocated cache at %p\n", - my_cache)); - // Now copy old cache into new cache - void **old_cache = ptr->addr; - for (int i = 0; i < __kmp_tp_capacity; ++i) { - my_cache[i] = old_cache[i]; - } - - // Add address of new my_cache to linked list for cleanup later - kmp_cached_addr_t *tp_cache_addr; - tp_cache_addr = (kmp_cached_addr_t *)&my_cache[newCapacity]; - tp_cache_addr->addr = my_cache; - tp_cache_addr->data = ptr->data; - tp_cache_addr->compiler_cache = ptr->compiler_cache; - tp_cache_addr->next = __kmp_threadpriv_cache_list; - __kmp_threadpriv_cache_list = tp_cache_addr; - - // Copy new cache to compiler's location: We can copy directly - // to (*compiler_cache) if compiler guarantees it will keep - // using the same location for the cache. This is not yet true - // for some compilers, in which case we have to check if - // compiler_cache is still pointing at old cache, and if so, we - // can point it at the new cache with an atomic compare&swap - // operation. (Old method will always work, but we should shift - // to new method (commented line below) when Intel and Clang - // compilers use new method.) - (void)KMP_COMPARE_AND_STORE_PTR(tp_cache_addr->compiler_cache, old_cache, - my_cache); - // TCW_PTR(*(tp_cache_addr->compiler_cache), my_cache); - - // If the store doesn't happen here, the compiler's old behavior will - // inevitably call __kmpc_threadprivate_cache with a new location for the - // cache, and that function will store the resized cache there at that - // point. - - // Nullify old cache's data pointer so we skip it next time - ptr->data = NULL; - } - ptr = ptr->next; - } - // After all caches are resized, update __kmp_tp_capacity to the new size - *(volatile int *)&__kmp_tp_capacity = newCapacity; -} - -/*! - @ingroup THREADPRIVATE - @param loc source location information - @param data pointer to data being privatized - @param ctor pointer to constructor function for data - @param cctor pointer to copy constructor function for data - @param dtor pointer to destructor function for data - @param vector_length length of the vector (bytes or elements?) - Register vector constructors and destructors for thread private data. -*/ -void __kmpc_threadprivate_register_vec(ident_t *loc, void *data, - kmpc_ctor_vec ctor, kmpc_cctor_vec cctor, - kmpc_dtor_vec dtor, - size_t vector_length) { - struct shared_common *d_tn, **lnk_tn; - - KC_TRACE(10, ("__kmpc_threadprivate_register_vec: called\n")); - -#ifdef USE_CHECKS_COMMON - /* copy constructor must be zero for current code gen (Nov 2002 - jph) */ - KMP_ASSERT(cctor == 0); -#endif /* USE_CHECKS_COMMON */ - - d_tn = __kmp_find_shared_task_common( - &__kmp_threadprivate_d_table, -1, - data); /* Only the global data table exists. */ - - if (d_tn == 0) { - d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); - d_tn->gbl_addr = data; - - d_tn->ct.ctorv = ctor; - d_tn->cct.cctorv = cctor; - d_tn->dt.dtorv = dtor; - d_tn->is_vec = TRUE; - d_tn->vec_len = (size_t)vector_length; - // d_tn->obj_init = 0; // AC: __kmp_allocate zeroes the memory - // d_tn->pod_init = 0; - lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]); - - d_tn->next = *lnk_tn; - *lnk_tn = d_tn; - } -} - -void __kmp_cleanup_threadprivate_caches() { - kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list; - - while (ptr) { - void **cache = ptr->addr; - __kmp_threadpriv_cache_list = ptr->next; - if (*ptr->compiler_cache) - *ptr->compiler_cache = NULL; - ptr->compiler_cache = NULL; - ptr->data = NULL; - ptr->addr = NULL; - ptr->next = NULL; - // Threadprivate data pointed at by cache entries are destroyed at end of - // __kmp_launch_thread with __kmp_common_destroy_gtid. - __kmp_free(cache); // implicitly frees ptr too - ptr = __kmp_threadpriv_cache_list; - } -} |
