summaryrefslogtreecommitdiffstats
path: root/contrib/libs/cxxsupp/openmp/kmp_lock.h
diff options
context:
space:
mode:
authorarcadia-devtools <[email protected]>2022-03-01 22:49:23 +0300
committerarcadia-devtools <[email protected]>2022-03-01 22:49:23 +0300
commitf1db7e1d2a6f1e911c41352aecb7897b8cc48d74 (patch)
treee551b29a4f1a5e46cb6f23b04e3192dcf5f9da32 /contrib/libs/cxxsupp/openmp/kmp_lock.h
parent8de79fac61fafe1e9e559da116135cca3f5846d1 (diff)
intermediate changes
ref:51d474bda1b99a2cf73ca7da0cd5398ef5683bf4
Diffstat (limited to 'contrib/libs/cxxsupp/openmp/kmp_lock.h')
-rw-r--r--contrib/libs/cxxsupp/openmp/kmp_lock.h1536
1 files changed, 767 insertions, 769 deletions
diff --git a/contrib/libs/cxxsupp/openmp/kmp_lock.h b/contrib/libs/cxxsupp/openmp/kmp_lock.h
index 4f6ad6414e5..8cd01d39812 100644
--- a/contrib/libs/cxxsupp/openmp/kmp_lock.h
+++ b/contrib/libs/cxxsupp/openmp/kmp_lock.h
@@ -2,26 +2,27 @@
* kmp_lock.h -- lock header file
*/
+
//===----------------------------------------------------------------------===//
//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//
+
#ifndef KMP_LOCK_H
#define KMP_LOCK_H
-#include <limits.h> // CHAR_BIT
-#include <stddef.h> // offsetof
+#include <limits.h> // CHAR_BIT
+#include <stddef.h> // offsetof
-#include "kmp_debug.h"
#include "kmp_os.h"
+#include "kmp_debug.h"
#ifdef __cplusplus
-#include <atomic>
-
extern "C" {
#endif // __cplusplus
@@ -29,8 +30,7 @@ extern "C" {
// Have to copy these definitions from kmp.h because kmp.h cannot be included
// due to circular dependencies. Will undef these at end of file.
-#define KMP_PAD(type, sz) \
- (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
+#define KMP_PAD(type, sz) (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
#define KMP_GTID_DNE (-2)
// Forward declaration of ident and ident_t
@@ -41,68 +41,86 @@ typedef struct ident ident_t;
// End of copied code.
// ----------------------------------------------------------------------------
+//
// We need to know the size of the area we can assume that the compiler(s)
-// allocated for objects of type omp_lock_t and omp_nest_lock_t. The Intel
+// allocated for obects of type omp_lock_t and omp_nest_lock_t. The Intel
// compiler always allocates a pointer-sized area, as does visual studio.
//
// gcc however, only allocates 4 bytes for regular locks, even on 64-bit
// intel archs. It allocates at least 8 bytes for nested lock (more on
// recent versions), but we are bounded by the pointer-sized chunks that
// the Intel compiler allocates.
+//
#if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT)
-#define OMP_LOCK_T_SIZE sizeof(int)
-#define OMP_NEST_LOCK_T_SIZE sizeof(void *)
+# define OMP_LOCK_T_SIZE sizeof(int)
+# define OMP_NEST_LOCK_T_SIZE sizeof(void *)
#else
-#define OMP_LOCK_T_SIZE sizeof(void *)
-#define OMP_NEST_LOCK_T_SIZE sizeof(void *)
+# define OMP_LOCK_T_SIZE sizeof(void *)
+# define OMP_NEST_LOCK_T_SIZE sizeof(void *)
#endif
+//
// The Intel compiler allocates a 32-byte chunk for a critical section.
// Both gcc and visual studio only allocate enough space for a pointer.
// Sometimes we know that the space was allocated by the Intel compiler.
-#define OMP_CRITICAL_SIZE sizeof(void *)
-#define INTEL_CRITICAL_SIZE 32
+//
+#define OMP_CRITICAL_SIZE sizeof(void *)
+#define INTEL_CRITICAL_SIZE 32
+//
// lock flags
+//
typedef kmp_uint32 kmp_lock_flags_t;
#define kmp_lf_critical_section 1
+//
// When a lock table is used, the indices are of kmp_lock_index_t
+//
typedef kmp_uint32 kmp_lock_index_t;
+//
// When memory allocated for locks are on the lock pool (free list),
// it is treated as structs of this type.
+//
struct kmp_lock_pool {
- union kmp_user_lock *next;
- kmp_lock_index_t index;
+ union kmp_user_lock *next;
+ kmp_lock_index_t index;
};
typedef struct kmp_lock_pool kmp_lock_pool_t;
-extern void __kmp_validate_locks(void);
+
+extern void __kmp_validate_locks( void );
+
// ----------------------------------------------------------------------------
+//
// There are 5 lock implementations:
+//
// 1. Test and set locks.
-// 2. futex locks (Linux* OS on x86 and
-// Intel(R) Many Integrated Core Architecture)
+// 2. futex locks (Linux* OS on x86 and Intel(R) Many Integrated Core architecture)
// 3. Ticket (Lamport bakery) locks.
// 4. Queuing locks (with separate spin fields).
// 5. DRPA (Dynamically Reconfigurable Distributed Polling Area) locks
//
// and 3 lock purposes:
-// 1. Bootstrap locks -- Used for a few locks available at library
-// startup-shutdown time.
+//
+// 1. Bootstrap locks -- Used for a few locks available at library startup-shutdown time.
// These do not require non-negative global thread ID's.
// 2. Internal RTL locks -- Used everywhere else in the RTL
// 3. User locks (includes critical sections)
+//
// ----------------------------------------------------------------------------
+
// ============================================================================
// Lock implementations.
-//
+// ============================================================================
+
+
+// ----------------------------------------------------------------------------
// Test and set locks.
//
// Non-nested test and set locks differ from the other lock kinds (except
@@ -113,56 +131,52 @@ extern void __kmp_validate_locks(void);
// bytes, so we have to use a lock table for nested locks, and avoid accessing
// the depth_locked field for non-nested locks.
//
-// Information normally available to the tools, such as lock location, lock
-// usage (normal lock vs. critical section), etc. is not available with test and
-// set locks.
+// Information normally available to the tools, such as lock location,
+// lock usage (normal lock vs. critical section), etc. is not available with
+// test and set locks.
// ----------------------------------------------------------------------------
struct kmp_base_tas_lock {
- // KMP_LOCK_FREE(tas) => unlocked; locked: (gtid+1) of owning thread
- std::atomic<kmp_int32> poll;
- kmp_int32 depth_locked; // depth locked, for nested locks only
+ volatile kmp_int32 poll; // 0 => unlocked
+ // locked: (gtid+1) of owning thread
+ kmp_int32 depth_locked; // depth locked, for nested locks only
};
typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
union kmp_tas_lock {
- kmp_base_tas_lock_t lk;
- kmp_lock_pool_t pool; // make certain struct is large enough
- double lk_align; // use worst case alignment; no cache line padding
+ kmp_base_tas_lock_t lk;
+ kmp_lock_pool_t pool; // make certain struct is large enough
+ double lk_align; // use worst case alignment
+ // no cache line padding
};
typedef union kmp_tas_lock kmp_tas_lock_t;
+//
// Static initializer for test and set lock variables. Usage:
// kmp_tas_lock_t xlock = KMP_TAS_LOCK_INITIALIZER( xlock );
-#define KMP_TAS_LOCK_INITIALIZER(lock) \
- { \
- { ATOMIC_VAR_INIT(KMP_LOCK_FREE(tas)), 0 } \
- }
-
-extern int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
-extern void __kmp_init_tas_lock(kmp_tas_lock_t *lck);
-extern void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck);
-
-extern int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
-extern void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck);
-extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
-
-#define KMP_LOCK_RELEASED 1
-#define KMP_LOCK_STILL_HELD 0
+//
+#define KMP_TAS_LOCK_INITIALIZER( lock ) { { 0, 0 } }
+
+extern int __kmp_acquire_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_tas_lock( kmp_tas_lock_t *lck );
+extern void __kmp_destroy_tas_lock( kmp_tas_lock_t *lck );
+
+extern int __kmp_acquire_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_nested_tas_lock( kmp_tas_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_nested_tas_lock( kmp_tas_lock_t *lck );
+extern void __kmp_destroy_nested_tas_lock( kmp_tas_lock_t *lck );
+
+#define KMP_LOCK_RELEASED 1
+#define KMP_LOCK_STILL_HELD 0
#define KMP_LOCK_ACQUIRED_FIRST 1
-#define KMP_LOCK_ACQUIRED_NEXT 0
-#ifndef KMP_USE_FUTEX
-#define KMP_USE_FUTEX \
- (KMP_OS_LINUX && \
- (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
-#endif
-#if KMP_USE_FUTEX
+#define KMP_LOCK_ACQUIRED_NEXT 0
+
+#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
// ----------------------------------------------------------------------------
// futex locks. futex locks are only available on Linux* OS.
@@ -170,138 +184,101 @@ extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
// Like non-nested test and set lock, non-nested futex locks use the memory
// allocated by the compiler for the lock, rather than a pointer to it.
//
-// Information normally available to the tools, such as lock location, lock
-// usage (normal lock vs. critical section), etc. is not available with test and
-// set locks. With non-nested futex locks, the lock owner is not even available.
+// Information normally available to the tools, such as lock location,
+// lock usage (normal lock vs. critical section), etc. is not available with
+// test and set locks. With non-nested futex locks, the lock owner is not
+// even available.
// ----------------------------------------------------------------------------
struct kmp_base_futex_lock {
- volatile kmp_int32 poll; // KMP_LOCK_FREE(futex) => unlocked
- // 2*(gtid+1) of owning thread, 0 if unlocked
- // locked: (gtid+1) of owning thread
- kmp_int32 depth_locked; // depth locked, for nested locks only
+ volatile kmp_int32 poll; // 0 => unlocked
+ // 2*(gtid+1) of owning thread, 0 if unlocked
+ // locked: (gtid+1) of owning thread
+ kmp_int32 depth_locked; // depth locked, for nested locks only
};
typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
union kmp_futex_lock {
- kmp_base_futex_lock_t lk;
- kmp_lock_pool_t pool; // make certain struct is large enough
- double lk_align; // use worst case alignment
- // no cache line padding
+ kmp_base_futex_lock_t lk;
+ kmp_lock_pool_t pool; // make certain struct is large enough
+ double lk_align; // use worst case alignment
+ // no cache line padding
};
typedef union kmp_futex_lock kmp_futex_lock_t;
+//
// Static initializer for futex lock variables. Usage:
// kmp_futex_lock_t xlock = KMP_FUTEX_LOCK_INITIALIZER( xlock );
-#define KMP_FUTEX_LOCK_INITIALIZER(lock) \
- { \
- { KMP_LOCK_FREE(futex), 0 } \
- }
-
-extern int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
-extern void __kmp_init_futex_lock(kmp_futex_lock_t *lck);
-extern void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck);
-
-extern int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck,
- kmp_int32 gtid);
-extern void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck);
-extern void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck);
-
-#endif // KMP_USE_FUTEX
-
-// ----------------------------------------------------------------------------
-// Ticket locks.
+//
+#define KMP_FUTEX_LOCK_INITIALIZER( lock ) { { 0, 0 } }
-#ifdef __cplusplus
+extern int __kmp_acquire_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_futex_lock( kmp_futex_lock_t *lck );
+extern void __kmp_destroy_futex_lock( kmp_futex_lock_t *lck );
-#ifdef _MSC_VER
-// MSVC won't allow use of std::atomic<> in a union since it has non-trivial
-// copy constructor.
+extern int __kmp_acquire_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_nested_futex_lock( kmp_futex_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_nested_futex_lock( kmp_futex_lock_t *lck );
+extern void __kmp_destroy_nested_futex_lock( kmp_futex_lock_t *lck );
-struct kmp_base_ticket_lock {
- // `initialized' must be the first entry in the lock data structure!
- std::atomic_bool initialized;
- volatile union kmp_ticket_lock *self; // points to the lock union
- ident_t const *location; // Source code location of omp_init_lock().
- std::atomic_uint
- next_ticket; // ticket number to give to next thread which acquires
- std::atomic_uint now_serving; // ticket number for thread which holds the lock
- std::atomic_int owner_id; // (gtid+1) of owning thread, 0 if unlocked
- std::atomic_int depth_locked; // depth locked, for nested locks only
- kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
-};
-#else
-struct kmp_base_ticket_lock {
- // `initialized' must be the first entry in the lock data structure!
- std::atomic<bool> initialized;
- volatile union kmp_ticket_lock *self; // points to the lock union
- ident_t const *location; // Source code location of omp_init_lock().
- std::atomic<unsigned>
- next_ticket; // ticket number to give to next thread which acquires
- std::atomic<unsigned>
- now_serving; // ticket number for thread which holds the lock
- std::atomic<int> owner_id; // (gtid+1) of owning thread, 0 if unlocked
- std::atomic<int> depth_locked; // depth locked, for nested locks only
- kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
-};
-#endif
+#endif // KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
-#else // __cplusplus
-struct kmp_base_ticket_lock;
+// ----------------------------------------------------------------------------
+// Ticket locks.
+// ----------------------------------------------------------------------------
-#endif // !__cplusplus
+struct kmp_base_ticket_lock {
+ // `initialized' must be the first entry in the lock data structure!
+ volatile union kmp_ticket_lock * initialized; // points to the lock union if in initialized state
+ ident_t const * location; // Source code location of omp_init_lock().
+ volatile kmp_uint32 next_ticket; // ticket number to give to next thread which acquires
+ volatile kmp_uint32 now_serving; // ticket number for thread which holds the lock
+ volatile kmp_int32 owner_id; // (gtid+1) of owning thread, 0 if unlocked
+ kmp_int32 depth_locked; // depth locked, for nested locks only
+ kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
+};
typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
union KMP_ALIGN_CACHE kmp_ticket_lock {
- kmp_base_ticket_lock_t
- lk; // This field must be first to allow static initializing.
- kmp_lock_pool_t pool;
- double lk_align; // use worst case alignment
- char lk_pad[KMP_PAD(kmp_base_ticket_lock_t, CACHE_LINE)];
+ kmp_base_ticket_lock_t lk; // This field must be first to allow static initializing.
+ kmp_lock_pool_t pool;
+ double lk_align; // use worst case alignment
+ char lk_pad[ KMP_PAD( kmp_base_ticket_lock_t, CACHE_LINE ) ];
};
typedef union kmp_ticket_lock kmp_ticket_lock_t;
+//
// Static initializer for simple ticket lock variables. Usage:
// kmp_ticket_lock_t xlock = KMP_TICKET_LOCK_INITIALIZER( xlock );
// Note the macro argument. It is important to make var properly initialized.
-#define KMP_TICKET_LOCK_INITIALIZER(lock) \
- { \
- { \
- ATOMIC_VAR_INIT(true) \
- , &(lock), NULL, ATOMIC_VAR_INIT(0U), ATOMIC_VAR_INIT(0U), \
- ATOMIC_VAR_INIT(0), ATOMIC_VAR_INIT(-1) \
- } \
- }
-
-extern int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_ticket_lock_with_cheks(kmp_ticket_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
-extern void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck);
-extern void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck);
-
-extern int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck,
- kmp_int32 gtid);
-extern void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck);
-extern void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck);
+//
+#define KMP_TICKET_LOCK_INITIALIZER( lock ) { { (kmp_ticket_lock_t *) & (lock), NULL, 0, 0, 0, -1 } }
+
+extern int __kmp_acquire_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_ticket_lock_with_cheks( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_ticket_lock( kmp_ticket_lock_t *lck );
+extern void __kmp_destroy_ticket_lock( kmp_ticket_lock_t *lck );
+
+extern int __kmp_acquire_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_nested_ticket_lock( kmp_ticket_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_nested_ticket_lock( kmp_ticket_lock_t *lck );
+extern void __kmp_destroy_nested_ticket_lock( kmp_ticket_lock_t *lck );
+
// ----------------------------------------------------------------------------
// Queuing locks.
+// ----------------------------------------------------------------------------
#if KMP_USE_ADAPTIVE_LOCKS
@@ -312,17 +289,17 @@ typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
#if KMP_DEBUG_ADAPTIVE_LOCKS
struct kmp_adaptive_lock_statistics {
- /* So we can get stats from locks that haven't been destroyed. */
- kmp_adaptive_lock_info_t *next;
- kmp_adaptive_lock_info_t *prev;
-
- /* Other statistics */
- kmp_uint32 successfulSpeculations;
- kmp_uint32 hardFailedSpeculations;
- kmp_uint32 softFailedSpeculations;
- kmp_uint32 nonSpeculativeAcquires;
- kmp_uint32 nonSpeculativeAcquireAttempts;
- kmp_uint32 lemmingYields;
+ /* So we can get stats from locks that haven't been destroyed. */
+ kmp_adaptive_lock_info_t * next;
+ kmp_adaptive_lock_info_t * prev;
+
+ /* Other statistics */
+ kmp_uint32 successfulSpeculations;
+ kmp_uint32 hardFailedSpeculations;
+ kmp_uint32 softFailedSpeculations;
+ kmp_uint32 nonSpeculativeAcquires;
+ kmp_uint32 nonSpeculativeAcquireAttempts;
+ kmp_uint32 lemmingYields;
};
typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
@@ -332,182 +309,188 @@ extern void __kmp_init_speculative_stats();
#endif // KMP_DEBUG_ADAPTIVE_LOCKS
-struct kmp_adaptive_lock_info {
- /* Values used for adaptivity.
- Although these are accessed from multiple threads we don't access them
- atomically, because if we miss updates it probably doesn't matter much. (It
- just affects our decision about whether to try speculation on the lock). */
- kmp_uint32 volatile badness;
- kmp_uint32 volatile acquire_attempts;
- /* Parameters of the lock. */
- kmp_uint32 max_badness;
- kmp_uint32 max_soft_retries;
+struct kmp_adaptive_lock_info
+{
+ /* Values used for adaptivity.
+ * Although these are accessed from multiple threads we don't access them atomically,
+ * because if we miss updates it probably doesn't matter much. (It just affects our
+ * decision about whether to try speculation on the lock).
+ */
+ kmp_uint32 volatile badness;
+ kmp_uint32 volatile acquire_attempts;
+ /* Parameters of the lock. */
+ kmp_uint32 max_badness;
+ kmp_uint32 max_soft_retries;
#if KMP_DEBUG_ADAPTIVE_LOCKS
- kmp_adaptive_lock_statistics_t volatile stats;
+ kmp_adaptive_lock_statistics_t volatile stats;
#endif
};
#endif // KMP_USE_ADAPTIVE_LOCKS
+
struct kmp_base_queuing_lock {
- // `initialized' must be the first entry in the lock data structure!
- volatile union kmp_queuing_lock
- *initialized; // Points to the lock union if in initialized state.
-
- ident_t const *location; // Source code location of omp_init_lock().
-
- KMP_ALIGN(8) // tail_id must be 8-byte aligned!
-
- volatile kmp_int32
- tail_id; // (gtid+1) of thread at tail of wait queue, 0 if empty
- // Must be no padding here since head/tail used in 8-byte CAS
- volatile kmp_int32
- head_id; // (gtid+1) of thread at head of wait queue, 0 if empty
- // Decl order assumes little endian
- // bakery-style lock
- volatile kmp_uint32
- next_ticket; // ticket number to give to next thread which acquires
- volatile kmp_uint32
- now_serving; // ticket number for thread which holds the lock
- volatile kmp_int32 owner_id; // (gtid+1) of owning thread, 0 if unlocked
- kmp_int32 depth_locked; // depth locked, for nested locks only
-
- kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
+ // `initialized' must be the first entry in the lock data structure!
+ volatile union kmp_queuing_lock *initialized; // Points to the lock union if in initialized state.
+
+ ident_t const * location; // Source code location of omp_init_lock().
+
+ KMP_ALIGN( 8 ) // tail_id must be 8-byte aligned!
+
+ volatile kmp_int32 tail_id; // (gtid+1) of thread at tail of wait queue, 0 if empty
+ // Must be no padding here since head/tail used in 8-byte CAS
+ volatile kmp_int32 head_id; // (gtid+1) of thread at head of wait queue, 0 if empty
+ // Decl order assumes little endian
+ // bakery-style lock
+ volatile kmp_uint32 next_ticket; // ticket number to give to next thread which acquires
+ volatile kmp_uint32 now_serving; // ticket number for thread which holds the lock
+ volatile kmp_int32 owner_id; // (gtid+1) of owning thread, 0 if unlocked
+ kmp_int32 depth_locked; // depth locked, for nested locks only
+
+ kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
};
typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
-KMP_BUILD_ASSERT(offsetof(kmp_base_queuing_lock_t, tail_id) % 8 == 0);
+KMP_BUILD_ASSERT( offsetof( kmp_base_queuing_lock_t, tail_id ) % 8 == 0 );
union KMP_ALIGN_CACHE kmp_queuing_lock {
- kmp_base_queuing_lock_t
- lk; // This field must be first to allow static initializing.
- kmp_lock_pool_t pool;
- double lk_align; // use worst case alignment
- char lk_pad[KMP_PAD(kmp_base_queuing_lock_t, CACHE_LINE)];
+ kmp_base_queuing_lock_t lk; // This field must be first to allow static initializing.
+ kmp_lock_pool_t pool;
+ double lk_align; // use worst case alignment
+ char lk_pad[ KMP_PAD( kmp_base_queuing_lock_t, CACHE_LINE ) ];
};
typedef union kmp_queuing_lock kmp_queuing_lock_t;
-extern int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
-extern void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck);
-extern void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck);
-
-extern int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck,
- kmp_int32 gtid);
-extern void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck);
-extern void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck);
+extern int __kmp_acquire_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_queuing_lock( kmp_queuing_lock_t *lck );
+extern void __kmp_destroy_queuing_lock( kmp_queuing_lock_t *lck );
+
+extern int __kmp_acquire_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_nested_queuing_lock( kmp_queuing_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_nested_queuing_lock( kmp_queuing_lock_t *lck );
+extern void __kmp_destroy_nested_queuing_lock( kmp_queuing_lock_t *lck );
#if KMP_USE_ADAPTIVE_LOCKS
// ----------------------------------------------------------------------------
// Adaptive locks.
+// ----------------------------------------------------------------------------
struct kmp_base_adaptive_lock {
- kmp_base_queuing_lock qlk;
- KMP_ALIGN(CACHE_LINE)
- kmp_adaptive_lock_info_t
- adaptive; // Information for the speculative adaptive lock
+ kmp_base_queuing_lock qlk;
+ KMP_ALIGN(CACHE_LINE)
+ kmp_adaptive_lock_info_t adaptive; // Information for the speculative adaptive lock
};
typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
union KMP_ALIGN_CACHE kmp_adaptive_lock {
- kmp_base_adaptive_lock_t lk;
- kmp_lock_pool_t pool;
- double lk_align;
- char lk_pad[KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE)];
+ kmp_base_adaptive_lock_t lk;
+ kmp_lock_pool_t pool;
+ double lk_align;
+ char lk_pad[ KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE) ];
};
typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
-#define GET_QLK_PTR(l) ((kmp_queuing_lock_t *)&(l)->lk.qlk)
+# define GET_QLK_PTR(l) ((kmp_queuing_lock_t *) & (l)->lk.qlk)
#endif // KMP_USE_ADAPTIVE_LOCKS
// ----------------------------------------------------------------------------
// DRDPA ticket locks.
+// ----------------------------------------------------------------------------
+
struct kmp_base_drdpa_lock {
- // All of the fields on the first cache line are only written when
- // initializing or reconfiguring the lock. These are relatively rare
- // operations, so data from the first cache line will usually stay resident in
- // the cache of each thread trying to acquire the lock.
- //
- // initialized must be the first entry in the lock data structure!
- KMP_ALIGN_CACHE
-
- volatile union kmp_drdpa_lock
- *initialized; // points to the lock union if in initialized state
- ident_t const *location; // Source code location of omp_init_lock().
- std::atomic<std::atomic<kmp_uint64> *> polls;
- std::atomic<kmp_uint64> mask; // is 2**num_polls-1 for mod op
- kmp_uint64 cleanup_ticket; // thread with cleanup ticket
- std::atomic<kmp_uint64> *old_polls; // will deallocate old_polls
- kmp_uint32 num_polls; // must be power of 2
-
- // next_ticket it needs to exist in a separate cache line, as it is
- // invalidated every time a thread takes a new ticket.
- KMP_ALIGN_CACHE
-
- std::atomic<kmp_uint64> next_ticket;
-
- // now_serving is used to store our ticket value while we hold the lock. It
- // has a slightly different meaning in the DRDPA ticket locks (where it is
- // written by the acquiring thread) than it does in the simple ticket locks
- // (where it is written by the releasing thread).
- //
- // Since now_serving is only read and written in the critical section,
- // it is non-volatile, but it needs to exist on a separate cache line,
- // as it is invalidated at every lock acquire.
- //
- // Likewise, the vars used for nested locks (owner_id and depth_locked) are
- // only written by the thread owning the lock, so they are put in this cache
- // line. owner_id is read by other threads, so it must be declared volatile.
- KMP_ALIGN_CACHE
- kmp_uint64 now_serving; // doesn't have to be volatile
- volatile kmp_uint32 owner_id; // (gtid+1) of owning thread, 0 if unlocked
- kmp_int32 depth_locked; // depth locked
- kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
+ //
+ // All of the fields on the first cache line are only written when
+ // initializing or reconfiguring the lock. These are relatively rare
+ // operations, so data from the first cache line will usually stay
+ // resident in the cache of each thread trying to acquire the lock.
+ //
+ // initialized must be the first entry in the lock data structure!
+ //
+ KMP_ALIGN_CACHE
+
+ volatile union kmp_drdpa_lock * initialized; // points to the lock union if in initialized state
+ ident_t const * location; // Source code location of omp_init_lock().
+ volatile struct kmp_lock_poll {
+ kmp_uint64 poll;
+ } * volatile polls;
+ volatile kmp_uint64 mask; // is 2**num_polls-1 for mod op
+ kmp_uint64 cleanup_ticket; // thread with cleanup ticket
+ volatile struct kmp_lock_poll * old_polls; // will deallocate old_polls
+ kmp_uint32 num_polls; // must be power of 2
+
+ //
+ // next_ticket it needs to exist in a separate cache line, as it is
+ // invalidated every time a thread takes a new ticket.
+ //
+ KMP_ALIGN_CACHE
+
+ volatile kmp_uint64 next_ticket;
+
+ //
+ // now_serving is used to store our ticket value while we hold the lock.
+ // It has a slightly different meaning in the DRDPA ticket locks (where
+ // it is written by the acquiring thread) than it does in the simple
+ // ticket locks (where it is written by the releasing thread).
+ //
+ // Since now_serving is only read an written in the critical section,
+ // it is non-volatile, but it needs to exist on a separate cache line,
+ // as it is invalidated at every lock acquire.
+ //
+ // Likewise, the vars used for nested locks (owner_id and depth_locked)
+ // are only written by the thread owning the lock, so they are put in
+ // this cache line. owner_id is read by other threads, so it must be
+ // declared volatile.
+ //
+ KMP_ALIGN_CACHE
+
+ kmp_uint64 now_serving; // doesn't have to be volatile
+ volatile kmp_uint32 owner_id; // (gtid+1) of owning thread, 0 if unlocked
+ kmp_int32 depth_locked; // depth locked
+ kmp_lock_flags_t flags; // lock specifics, e.g. critical section lock
};
typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
union KMP_ALIGN_CACHE kmp_drdpa_lock {
- kmp_base_drdpa_lock_t
- lk; // This field must be first to allow static initializing. */
- kmp_lock_pool_t pool;
- double lk_align; // use worst case alignment
- char lk_pad[KMP_PAD(kmp_base_drdpa_lock_t, CACHE_LINE)];
+ kmp_base_drdpa_lock_t lk; // This field must be first to allow static initializing. */
+ kmp_lock_pool_t pool;
+ double lk_align; // use worst case alignment
+ char lk_pad[ KMP_PAD( kmp_base_drdpa_lock_t, CACHE_LINE ) ];
};
typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
-extern int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
-extern void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck);
-extern void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck);
+extern int __kmp_acquire_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_drdpa_lock( kmp_drdpa_lock_t *lck );
+extern void __kmp_destroy_drdpa_lock( kmp_drdpa_lock_t *lck );
+
+extern int __kmp_acquire_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_test_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
+extern int __kmp_release_nested_drdpa_lock( kmp_drdpa_lock_t *lck, kmp_int32 gtid );
+extern void __kmp_init_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
+extern void __kmp_destroy_nested_drdpa_lock( kmp_drdpa_lock_t *lck );
-extern int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
- kmp_int32 gtid);
-extern int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
-extern int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
- kmp_int32 gtid);
-extern void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
-extern void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
// ============================================================================
// Lock purposes.
// ============================================================================
+
+// ----------------------------------------------------------------------------
// Bootstrap locks.
-//
+// ----------------------------------------------------------------------------
+
// Bootstrap locks -- very few locks used at library initialization time.
// Bootstrap locks are currently implemented as ticket locks.
// They could also be implemented as test and set lock, but cannot be
@@ -516,85 +499,111 @@ extern void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
-#define KMP_BOOTSTRAP_LOCK_INITIALIZER(lock) KMP_TICKET_LOCK_INITIALIZER((lock))
-#define KMP_BOOTSTRAP_LOCK_INIT(lock) \
- kmp_bootstrap_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock)
+#define KMP_BOOTSTRAP_LOCK_INITIALIZER( lock ) KMP_TICKET_LOCK_INITIALIZER( (lock) )
-static inline int __kmp_acquire_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
- return __kmp_acquire_ticket_lock(lck, KMP_GTID_DNE);
+static inline int
+__kmp_acquire_bootstrap_lock( kmp_bootstrap_lock_t *lck )
+{
+ return __kmp_acquire_ticket_lock( lck, KMP_GTID_DNE );
}
-static inline int __kmp_test_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
- return __kmp_test_ticket_lock(lck, KMP_GTID_DNE);
+static inline int
+__kmp_test_bootstrap_lock( kmp_bootstrap_lock_t *lck )
+{
+ return __kmp_test_ticket_lock( lck, KMP_GTID_DNE );
}
-static inline void __kmp_release_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
- __kmp_release_ticket_lock(lck, KMP_GTID_DNE);
+static inline void
+__kmp_release_bootstrap_lock( kmp_bootstrap_lock_t *lck )
+{
+ __kmp_release_ticket_lock( lck, KMP_GTID_DNE );
}
-static inline void __kmp_init_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
- __kmp_init_ticket_lock(lck);
+static inline void
+__kmp_init_bootstrap_lock( kmp_bootstrap_lock_t *lck )
+{
+ __kmp_init_ticket_lock( lck );
}
-static inline void __kmp_destroy_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
- __kmp_destroy_ticket_lock(lck);
+static inline void
+__kmp_destroy_bootstrap_lock( kmp_bootstrap_lock_t *lck )
+{
+ __kmp_destroy_ticket_lock( lck );
}
+
+// ----------------------------------------------------------------------------
// Internal RTL locks.
+// ----------------------------------------------------------------------------
+
//
// Internal RTL locks are also implemented as ticket locks, for now.
//
// FIXME - We should go through and figure out which lock kind works best for
// each internal lock, and use the type declaration and function calls for
// that explicit lock kind (and get rid of this section).
+//
typedef kmp_ticket_lock_t kmp_lock_t;
-#define KMP_LOCK_INIT(lock) kmp_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock)
-
-static inline int __kmp_acquire_lock(kmp_lock_t *lck, kmp_int32 gtid) {
- return __kmp_acquire_ticket_lock(lck, gtid);
+static inline int
+__kmp_acquire_lock( kmp_lock_t *lck, kmp_int32 gtid )
+{
+ return __kmp_acquire_ticket_lock( lck, gtid );
}
-static inline int __kmp_test_lock(kmp_lock_t *lck, kmp_int32 gtid) {
- return __kmp_test_ticket_lock(lck, gtid);
+static inline int
+__kmp_test_lock( kmp_lock_t *lck, kmp_int32 gtid )
+{
+ return __kmp_test_ticket_lock( lck, gtid );
}
-static inline void __kmp_release_lock(kmp_lock_t *lck, kmp_int32 gtid) {
- __kmp_release_ticket_lock(lck, gtid);
+static inline void
+__kmp_release_lock( kmp_lock_t *lck, kmp_int32 gtid )
+{
+ __kmp_release_ticket_lock( lck, gtid );
}
-static inline void __kmp_init_lock(kmp_lock_t *lck) {
- __kmp_init_ticket_lock(lck);
+static inline void
+__kmp_init_lock( kmp_lock_t *lck )
+{
+ __kmp_init_ticket_lock( lck );
}
-static inline void __kmp_destroy_lock(kmp_lock_t *lck) {
- __kmp_destroy_ticket_lock(lck);
+static inline void
+__kmp_destroy_lock( kmp_lock_t *lck )
+{
+ __kmp_destroy_ticket_lock( lck );
}
+
+// ----------------------------------------------------------------------------
// User locks.
+// ----------------------------------------------------------------------------
+
+//
+// Do not allocate objects of type union kmp_user_lock!!!
+// This will waste space unless __kmp_user_lock_kind == lk_drdpa.
+// Instead, check the value of __kmp_user_lock_kind and allocate objects of
+// the type of the appropriate union member, and cast their addresses to
+// kmp_user_lock_p.
//
-// Do not allocate objects of type union kmp_user_lock!!! This will waste space
-// unless __kmp_user_lock_kind == lk_drdpa. Instead, check the value of
-// __kmp_user_lock_kind and allocate objects of the type of the appropriate
-// union member, and cast their addresses to kmp_user_lock_p.
enum kmp_lock_kind {
- lk_default = 0,
- lk_tas,
-#if KMP_USE_FUTEX
- lk_futex,
+ lk_default = 0,
+ lk_tas,
+#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+ lk_futex,
#endif
#if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX
- lk_hle,
- lk_rtm_queuing,
- lk_rtm_spin,
+ lk_hle,
+ lk_rtm,
#endif
- lk_ticket,
- lk_queuing,
- lk_drdpa,
+ lk_ticket,
+ lk_queuing,
+ lk_drdpa,
#if KMP_USE_ADAPTIVE_LOCKS
- lk_adaptive
+ lk_adaptive
#endif // KMP_USE_ADAPTIVE_LOCKS
};
@@ -603,253 +612,279 @@ typedef enum kmp_lock_kind kmp_lock_kind_t;
extern kmp_lock_kind_t __kmp_user_lock_kind;
union kmp_user_lock {
- kmp_tas_lock_t tas;
-#if KMP_USE_FUTEX
- kmp_futex_lock_t futex;
+ kmp_tas_lock_t tas;
+#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+ kmp_futex_lock_t futex;
#endif
- kmp_ticket_lock_t ticket;
- kmp_queuing_lock_t queuing;
- kmp_drdpa_lock_t drdpa;
+ kmp_ticket_lock_t ticket;
+ kmp_queuing_lock_t queuing;
+ kmp_drdpa_lock_t drdpa;
#if KMP_USE_ADAPTIVE_LOCKS
- kmp_adaptive_lock_t adaptive;
+ kmp_adaptive_lock_t adaptive;
#endif // KMP_USE_ADAPTIVE_LOCKS
- kmp_lock_pool_t pool;
+ kmp_lock_pool_t pool;
};
typedef union kmp_user_lock *kmp_user_lock_p;
-#if !KMP_USE_DYNAMIC_LOCK
+#if ! KMP_USE_DYNAMIC_LOCK
extern size_t __kmp_base_user_lock_size;
extern size_t __kmp_user_lock_size;
-extern kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck);
+extern kmp_int32 ( *__kmp_get_user_lock_owner_ )( kmp_user_lock_p lck );
-static inline kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck) {
- KMP_DEBUG_ASSERT(__kmp_get_user_lock_owner_ != NULL);
- return (*__kmp_get_user_lock_owner_)(lck);
+static inline kmp_int32
+__kmp_get_user_lock_owner( kmp_user_lock_p lck )
+{
+ KMP_DEBUG_ASSERT( __kmp_get_user_lock_owner_ != NULL );
+ return ( *__kmp_get_user_lock_owner_ )( lck );
}
-extern int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
- kmp_int32 gtid);
-
-#if KMP_OS_LINUX && \
- (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
-
-#define __kmp_acquire_user_lock_with_checks(lck, gtid) \
- if (__kmp_user_lock_kind == lk_tas) { \
- if (__kmp_env_consistency_check) { \
- char const *const func = "omp_set_lock"; \
- if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) && \
- lck->tas.lk.depth_locked != -1) { \
- KMP_FATAL(LockNestableUsedAsSimple, func); \
- } \
- if ((gtid >= 0) && (lck->tas.lk.poll - 1 == gtid)) { \
- KMP_FATAL(LockIsAlreadyOwned, func); \
- } \
- } \
- if (lck->tas.lk.poll != 0 || \
- !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \
- kmp_uint32 spins; \
- KMP_FSYNC_PREPARE(lck); \
- KMP_INIT_YIELD(spins); \
- do { \
- KMP_YIELD_OVERSUB_ELSE_SPIN(spins); \
- } while (lck->tas.lk.poll != 0 || !__kmp_atomic_compare_store_acq( \
- &lck->tas.lk.poll, 0, gtid + 1)); \
- } \
- KMP_FSYNC_ACQUIRED(lck); \
- } else { \
- KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL); \
- (*__kmp_acquire_user_lock_with_checks_)(lck, gtid); \
- }
+extern int ( *__kmp_acquire_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
+
+#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+
+#define __kmp_acquire_user_lock_with_checks(lck,gtid) \
+ if (__kmp_user_lock_kind == lk_tas) { \
+ if ( __kmp_env_consistency_check ) { \
+ char const * const func = "omp_set_lock"; \
+ if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE ) \
+ && lck->tas.lk.depth_locked != -1 ) { \
+ KMP_FATAL( LockNestableUsedAsSimple, func ); \
+ } \
+ if ( ( gtid >= 0 ) && ( lck->tas.lk.poll - 1 == gtid ) ) { \
+ KMP_FATAL( LockIsAlreadyOwned, func ); \
+ } \
+ } \
+ if ( ( lck->tas.lk.poll != 0 ) || \
+ ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
+ kmp_uint32 spins; \
+ KMP_FSYNC_PREPARE( lck ); \
+ KMP_INIT_YIELD( spins ); \
+ if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
+ KMP_YIELD( TRUE ); \
+ } else { \
+ KMP_YIELD_SPIN( spins ); \
+ } \
+ while ( ( lck->tas.lk.poll != 0 ) || \
+ ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
+ if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
+ KMP_YIELD( TRUE ); \
+ } else { \
+ KMP_YIELD_SPIN( spins ); \
+ } \
+ } \
+ } \
+ KMP_FSYNC_ACQUIRED( lck ); \
+ } else { \
+ KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL ); \
+ ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid ); \
+ }
#else
-static inline int __kmp_acquire_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL);
- return (*__kmp_acquire_user_lock_with_checks_)(lck, gtid);
+static inline int
+__kmp_acquire_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ KMP_DEBUG_ASSERT( __kmp_acquire_user_lock_with_checks_ != NULL );
+ return ( *__kmp_acquire_user_lock_with_checks_ )( lck, gtid );
}
#endif
-extern int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
- kmp_int32 gtid);
-
-#if KMP_OS_LINUX && \
- (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
-
-#include "kmp_i18n.h" /* AC: KMP_FATAL definition */
-extern int __kmp_env_consistency_check; /* AC: copy from kmp.h here */
-static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- if (__kmp_user_lock_kind == lk_tas) {
- if (__kmp_env_consistency_check) {
- char const *const func = "omp_test_lock";
- if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
- lck->tas.lk.depth_locked != -1) {
- KMP_FATAL(LockNestableUsedAsSimple, func);
- }
+extern int ( *__kmp_test_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
+
+#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
+
+#include "kmp_i18n.h" /* AC: KMP_FATAL definition */
+extern int __kmp_env_consistency_check; /* AC: copy from kmp.h here */
+static inline int
+__kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ if ( __kmp_user_lock_kind == lk_tas ) {
+ if ( __kmp_env_consistency_check ) {
+ char const * const func = "omp_test_lock";
+ if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_LOCK_T_SIZE )
+ && lck->tas.lk.depth_locked != -1 ) {
+ KMP_FATAL( LockNestableUsedAsSimple, func );
+ }
+ }
+ return ( ( lck->tas.lk.poll == 0 ) &&
+ KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
+ } else {
+ KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
+ return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
}
- return ((lck->tas.lk.poll == 0) &&
- __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
- } else {
- KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
- return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
- }
}
#else
-static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
- return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
+static inline int
+__kmp_test_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ KMP_DEBUG_ASSERT( __kmp_test_user_lock_with_checks_ != NULL );
+ return ( *__kmp_test_user_lock_with_checks_ )( lck, gtid );
}
#endif
-extern int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
- kmp_int32 gtid);
+extern int ( *__kmp_release_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
-static inline void __kmp_release_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- KMP_DEBUG_ASSERT(__kmp_release_user_lock_with_checks_ != NULL);
- (*__kmp_release_user_lock_with_checks_)(lck, gtid);
+static inline void
+__kmp_release_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ KMP_DEBUG_ASSERT( __kmp_release_user_lock_with_checks_ != NULL );
+ ( *__kmp_release_user_lock_with_checks_ ) ( lck, gtid );
}
-extern void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck);
+extern void ( *__kmp_init_user_lock_with_checks_ )( kmp_user_lock_p lck );
-static inline void __kmp_init_user_lock_with_checks(kmp_user_lock_p lck) {
- KMP_DEBUG_ASSERT(__kmp_init_user_lock_with_checks_ != NULL);
- (*__kmp_init_user_lock_with_checks_)(lck);
+static inline void
+__kmp_init_user_lock_with_checks( kmp_user_lock_p lck )
+{
+ KMP_DEBUG_ASSERT( __kmp_init_user_lock_with_checks_ != NULL );
+ ( *__kmp_init_user_lock_with_checks_ )( lck );
}
+//
// We need a non-checking version of destroy lock for when the RTL is
// doing the cleanup as it can't always tell if the lock is nested or not.
-extern void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck);
+//
+extern void ( *__kmp_destroy_user_lock_ )( kmp_user_lock_p lck );
-static inline void __kmp_destroy_user_lock(kmp_user_lock_p lck) {
- KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_ != NULL);
- (*__kmp_destroy_user_lock_)(lck);
+static inline void
+__kmp_destroy_user_lock( kmp_user_lock_p lck )
+{
+ KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_ != NULL );
+ ( *__kmp_destroy_user_lock_ )( lck );
}
-extern void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck);
+extern void ( *__kmp_destroy_user_lock_with_checks_ )( kmp_user_lock_p lck );
-static inline void __kmp_destroy_user_lock_with_checks(kmp_user_lock_p lck) {
- KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_with_checks_ != NULL);
- (*__kmp_destroy_user_lock_with_checks_)(lck);
+static inline void
+__kmp_destroy_user_lock_with_checks( kmp_user_lock_p lck )
+{
+ KMP_DEBUG_ASSERT( __kmp_destroy_user_lock_with_checks_ != NULL );
+ ( *__kmp_destroy_user_lock_with_checks_ )( lck );
}
-extern int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
- kmp_int32 gtid);
+extern int ( *__kmp_acquire_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
-#define __kmp_acquire_nested_user_lock_with_checks(lck, gtid, depth) \
- if (__kmp_user_lock_kind == lk_tas) { \
- if (__kmp_env_consistency_check) { \
- char const *const func = "omp_set_nest_lock"; \
- if ((sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) && \
- lck->tas.lk.depth_locked == -1) { \
- KMP_FATAL(LockSimpleUsedAsNestable, func); \
- } \
- } \
- if (lck->tas.lk.poll - 1 == gtid) { \
- lck->tas.lk.depth_locked += 1; \
- *depth = KMP_LOCK_ACQUIRED_NEXT; \
- } else { \
- if ((lck->tas.lk.poll != 0) || \
- !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \
- kmp_uint32 spins; \
- KMP_FSYNC_PREPARE(lck); \
- KMP_INIT_YIELD(spins); \
- do { \
- KMP_YIELD_OVERSUB_ELSE_SPIN(spins); \
- } while ( \
- (lck->tas.lk.poll != 0) || \
- !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)); \
- } \
- lck->tas.lk.depth_locked = 1; \
- *depth = KMP_LOCK_ACQUIRED_FIRST; \
- } \
- KMP_FSYNC_ACQUIRED(lck); \
- } else { \
- KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL); \
- *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid); \
- }
+#define __kmp_acquire_nested_user_lock_with_checks(lck,gtid,depth) \
+ if (__kmp_user_lock_kind == lk_tas) { \
+ if ( __kmp_env_consistency_check ) { \
+ char const * const func = "omp_set_nest_lock"; \
+ if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE ) \
+ && lck->tas.lk.depth_locked == -1 ) { \
+ KMP_FATAL( LockSimpleUsedAsNestable, func ); \
+ } \
+ } \
+ if ( lck->tas.lk.poll - 1 == gtid ) { \
+ lck->tas.lk.depth_locked += 1; \
+ *depth = KMP_LOCK_ACQUIRED_NEXT; \
+ } else { \
+ if ( ( lck->tas.lk.poll != 0 ) || \
+ ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
+ kmp_uint32 spins; \
+ KMP_FSYNC_PREPARE( lck ); \
+ KMP_INIT_YIELD( spins ); \
+ if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
+ KMP_YIELD( TRUE ); \
+ } else { \
+ KMP_YIELD_SPIN( spins ); \
+ } \
+ while ( ( lck->tas.lk.poll != 0 ) || \
+ ( ! KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) ) ) { \
+ if ( TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc) ) { \
+ KMP_YIELD( TRUE ); \
+ } else { \
+ KMP_YIELD_SPIN( spins ); \
+ } \
+ } \
+ } \
+ lck->tas.lk.depth_locked = 1; \
+ *depth = KMP_LOCK_ACQUIRED_FIRST; \
+ } \
+ KMP_FSYNC_ACQUIRED( lck ); \
+ } else { \
+ KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL ); \
+ *depth = ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid ); \
+ }
#else
static inline void
-__kmp_acquire_nested_user_lock_with_checks(kmp_user_lock_p lck, kmp_int32 gtid,
- int *depth) {
- KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL);
- *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid);
+__kmp_acquire_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid, int* depth )
+{
+ KMP_DEBUG_ASSERT( __kmp_acquire_nested_user_lock_with_checks_ != NULL );
+ *depth = ( *__kmp_acquire_nested_user_lock_with_checks_ )( lck, gtid );
}
#endif
-extern int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
- kmp_int32 gtid);
+extern int ( *__kmp_test_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
-static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- if (__kmp_user_lock_kind == lk_tas) {
- int retval;
- if (__kmp_env_consistency_check) {
- char const *const func = "omp_test_nest_lock";
- if ((sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) &&
- lck->tas.lk.depth_locked == -1) {
- KMP_FATAL(LockSimpleUsedAsNestable, func);
- }
- }
- KMP_DEBUG_ASSERT(gtid >= 0);
- if (lck->tas.lk.poll - 1 ==
- gtid) { /* __kmp_get_tas_lock_owner( lck ) == gtid */
- return ++lck->tas.lk.depth_locked; /* same owner, depth increased */
- }
- retval = ((lck->tas.lk.poll == 0) &&
- __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
- if (retval) {
- KMP_MB();
- lck->tas.lk.depth_locked = 1;
+static inline int
+__kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ if ( __kmp_user_lock_kind == lk_tas ) {
+ int retval;
+ if ( __kmp_env_consistency_check ) {
+ char const * const func = "omp_test_nest_lock";
+ if ( ( sizeof ( kmp_tas_lock_t ) <= OMP_NEST_LOCK_T_SIZE )
+ && lck->tas.lk.depth_locked == -1 ) {
+ KMP_FATAL( LockSimpleUsedAsNestable, func );
+ }
+ }
+ KMP_DEBUG_ASSERT( gtid >= 0 );
+ if ( lck->tas.lk.poll - 1 == gtid ) { /* __kmp_get_tas_lock_owner( lck ) == gtid */
+ return ++lck->tas.lk.depth_locked; /* same owner, depth increased */
+ }
+ retval = ( ( lck->tas.lk.poll == 0 ) &&
+ KMP_COMPARE_AND_STORE_ACQ32( &(lck->tas.lk.poll), 0, gtid + 1 ) );
+ if ( retval ) {
+ KMP_MB();
+ lck->tas.lk.depth_locked = 1;
+ }
+ return retval;
+ } else {
+ KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
+ return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
}
- return retval;
- } else {
- KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
- return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
- }
}
#else
-static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
- return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
+static inline int
+__kmp_test_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ KMP_DEBUG_ASSERT( __kmp_test_nested_user_lock_with_checks_ != NULL );
+ return ( *__kmp_test_nested_user_lock_with_checks_ )( lck, gtid );
}
#endif
-extern int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
- kmp_int32 gtid);
+extern int ( *__kmp_release_nested_user_lock_with_checks_ )( kmp_user_lock_p lck, kmp_int32 gtid );
static inline int
-__kmp_release_nested_user_lock_with_checks(kmp_user_lock_p lck,
- kmp_int32 gtid) {
- KMP_DEBUG_ASSERT(__kmp_release_nested_user_lock_with_checks_ != NULL);
- return (*__kmp_release_nested_user_lock_with_checks_)(lck, gtid);
+__kmp_release_nested_user_lock_with_checks( kmp_user_lock_p lck, kmp_int32 gtid )
+{
+ KMP_DEBUG_ASSERT( __kmp_release_nested_user_lock_with_checks_ != NULL );
+ return ( *__kmp_release_nested_user_lock_with_checks_ )( lck, gtid );
}
-extern void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
+extern void ( *__kmp_init_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
-static inline void
-__kmp_init_nested_user_lock_with_checks(kmp_user_lock_p lck) {
- KMP_DEBUG_ASSERT(__kmp_init_nested_user_lock_with_checks_ != NULL);
- (*__kmp_init_nested_user_lock_with_checks_)(lck);
+static inline void __kmp_init_nested_user_lock_with_checks( kmp_user_lock_p lck )
+{
+ KMP_DEBUG_ASSERT( __kmp_init_nested_user_lock_with_checks_ != NULL );
+ ( *__kmp_init_nested_user_lock_with_checks_ )( lck );
}
-extern void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
+extern void ( *__kmp_destroy_nested_user_lock_with_checks_ )( kmp_user_lock_p lck );
static inline void
-__kmp_destroy_nested_user_lock_with_checks(kmp_user_lock_p lck) {
- KMP_DEBUG_ASSERT(__kmp_destroy_nested_user_lock_with_checks_ != NULL);
- (*__kmp_destroy_nested_user_lock_with_checks_)(lck);
+__kmp_destroy_nested_user_lock_with_checks( kmp_user_lock_p lck )
+{
+ KMP_DEBUG_ASSERT( __kmp_destroy_nested_user_lock_with_checks_ != NULL );
+ ( *__kmp_destroy_nested_user_lock_with_checks_ )( lck );
}
+//
// user lock functions which do not necessarily exist for all lock kinds.
//
// The "set" functions usually have wrapper routines that check for a NULL set
@@ -861,97 +896,104 @@ __kmp_destroy_nested_user_lock_with_checks(kmp_user_lock_p lck) {
//
// In other cases, the calling code really should differentiate between an
// unimplemented function and one that is implemented but returning NULL /
-// invalid value. If this is the case, no get function wrapper exists.
+// invalied value. If this is the case, no get function wrapper exists.
+//
-extern int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck);
+extern int ( *__kmp_is_user_lock_initialized_ )( kmp_user_lock_p lck );
-// no set function; fields set during local allocation
+// no set function; fields set durining local allocation
-extern const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck);
+extern const ident_t * ( *__kmp_get_user_lock_location_ )( kmp_user_lock_p lck );
-static inline const ident_t *__kmp_get_user_lock_location(kmp_user_lock_p lck) {
- if (__kmp_get_user_lock_location_ != NULL) {
- return (*__kmp_get_user_lock_location_)(lck);
- } else {
- return NULL;
- }
+static inline const ident_t *
+__kmp_get_user_lock_location( kmp_user_lock_p lck )
+{
+ if ( __kmp_get_user_lock_location_ != NULL ) {
+ return ( *__kmp_get_user_lock_location_ )( lck );
+ }
+ else {
+ return NULL;
+ }
}
-extern void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
- const ident_t *loc);
+extern void ( *__kmp_set_user_lock_location_ )( kmp_user_lock_p lck, const ident_t *loc );
-static inline void __kmp_set_user_lock_location(kmp_user_lock_p lck,
- const ident_t *loc) {
- if (__kmp_set_user_lock_location_ != NULL) {
- (*__kmp_set_user_lock_location_)(lck, loc);
- }
+static inline void
+__kmp_set_user_lock_location( kmp_user_lock_p lck, const ident_t *loc )
+{
+ if ( __kmp_set_user_lock_location_ != NULL ) {
+ ( *__kmp_set_user_lock_location_ )( lck, loc );
+ }
}
-extern kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck);
+extern kmp_lock_flags_t ( *__kmp_get_user_lock_flags_ )( kmp_user_lock_p lck );
-extern void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
- kmp_lock_flags_t flags);
+extern void ( *__kmp_set_user_lock_flags_ )( kmp_user_lock_p lck, kmp_lock_flags_t flags );
-static inline void __kmp_set_user_lock_flags(kmp_user_lock_p lck,
- kmp_lock_flags_t flags) {
- if (__kmp_set_user_lock_flags_ != NULL) {
- (*__kmp_set_user_lock_flags_)(lck, flags);
- }
+static inline void
+__kmp_set_user_lock_flags( kmp_user_lock_p lck, kmp_lock_flags_t flags )
+{
+ if ( __kmp_set_user_lock_flags_ != NULL ) {
+ ( *__kmp_set_user_lock_flags_ )( lck, flags );
+ }
}
-// The function which sets up all of the vtbl pointers for kmp_user_lock_t.
-extern void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind);
+//
+// The fuction which sets up all of the vtbl pointers for kmp_user_lock_t.
+//
+extern void __kmp_set_user_lock_vptrs( kmp_lock_kind_t user_lock_kind );
+//
// Macros for binding user lock functions.
-#define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) \
- { \
- __kmp_acquire##nest##user_lock_with_checks_ = (int (*)( \
- kmp_user_lock_p, kmp_int32))__kmp_acquire##nest##kind##_##suffix; \
- __kmp_release##nest##user_lock_with_checks_ = (int (*)( \
- kmp_user_lock_p, kmp_int32))__kmp_release##nest##kind##_##suffix; \
- __kmp_test##nest##user_lock_with_checks_ = (int (*)( \
- kmp_user_lock_p, kmp_int32))__kmp_test##nest##kind##_##suffix; \
- __kmp_init##nest##user_lock_with_checks_ = \
- (void (*)(kmp_user_lock_p))__kmp_init##nest##kind##_##suffix; \
- __kmp_destroy##nest##user_lock_with_checks_ = \
- (void (*)(kmp_user_lock_p))__kmp_destroy##nest##kind##_##suffix; \
- }
-
-#define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock)
-#define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) \
- KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks)
-#define KMP_BIND_NESTED_USER_LOCK(kind) \
- KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock)
-#define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) \
- KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks)
+//
+#define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) { \
+ __kmp_acquire##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \
+ __kmp_acquire##nest##kind##_##suffix; \
+ __kmp_release##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \
+ __kmp_release##nest##kind##_##suffix; \
+ __kmp_test##nest##user_lock_with_checks_ = ( int (*)( kmp_user_lock_p, kmp_int32 ) ) \
+ __kmp_test##nest##kind##_##suffix; \
+ __kmp_init##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \
+ __kmp_init##nest##kind##_##suffix; \
+ __kmp_destroy##nest##user_lock_with_checks_ = ( void (*)( kmp_user_lock_p ) ) \
+ __kmp_destroy##nest##kind##_##suffix; \
+}
+#define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock)
+#define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks)
+#define KMP_BIND_NESTED_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock)
+#define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks)
+
+// ----------------------------------------------------------------------------
// User lock table & lock allocation
-/* On 64-bit Linux* OS (and OS X*) GNU compiler allocates only 4 bytems memory
- for lock variable, which is not enough to store a pointer, so we have to use
- lock indexes instead of pointers and maintain lock table to map indexes to
- pointers.
-
-
- Note: The first element of the table is not a pointer to lock! It is a
- pointer to previously allocated table (or NULL if it is the first table).
-
- Usage:
-
- if ( OMP_LOCK_T_SIZE < sizeof( <lock> ) ) { // or OMP_NEST_LOCK_T_SIZE
- Lock table is fully utilized. User locks are indexes, so table is used on
- user lock operation.
- Note: it may be the case (lin_32) that we don't need to use a lock
- table for regular locks, but do need the table for nested locks.
- }
- else {
- Lock table initialized but not actually used.
- }
+// ----------------------------------------------------------------------------
+
+/*
+ On 64-bit Linux* OS (and OS X*) GNU compiler allocates only 4 bytems memory for lock variable, which
+ is not enough to store a pointer, so we have to use lock indexes instead of pointers and
+ maintain lock table to map indexes to pointers.
+
+
+ Note: The first element of the table is not a pointer to lock! It is a pointer to previously
+ allocated table (or NULL if it is the first table).
+
+ Usage:
+
+ if ( OMP_LOCK_T_SIZE < sizeof( <lock> ) ) { // or OMP_NEST_LOCK_T_SIZE
+ Lock table is fully utilized. User locks are indexes, so table is
+ used on user lock operation.
+ Note: it may be the case (lin_32) that we don't need to use a lock
+ table for regular locks, but do need the table for nested locks.
+ }
+ else {
+ Lock table initialized but not actually used.
+ }
*/
struct kmp_lock_table {
- kmp_lock_index_t used; // Number of used elements
- kmp_lock_index_t allocated; // Number of allocated elements
- kmp_user_lock_p *table; // Lock table.
+ kmp_lock_index_t used; // Number of used elements
+ kmp_lock_index_t allocated; // Number of allocated elements
+ kmp_user_lock_p * table; // Lock table.
};
typedef struct kmp_lock_table kmp_lock_table_t;
@@ -960,8 +1002,8 @@ extern kmp_lock_table_t __kmp_user_lock_table;
extern kmp_user_lock_p __kmp_lock_pool;
struct kmp_block_of_locks {
- struct kmp_block_of_locks *next_block;
- void *locks;
+ struct kmp_block_of_locks * next_block;
+ void * locks;
};
typedef struct kmp_block_of_locks kmp_block_of_locks_t;
@@ -969,25 +1011,21 @@ typedef struct kmp_block_of_locks kmp_block_of_locks_t;
extern kmp_block_of_locks_t *__kmp_lock_blocks;
extern int __kmp_num_locks_in_block;
-extern kmp_user_lock_p __kmp_user_lock_allocate(void **user_lock,
- kmp_int32 gtid,
- kmp_lock_flags_t flags);
-extern void __kmp_user_lock_free(void **user_lock, kmp_int32 gtid,
- kmp_user_lock_p lck);
-extern kmp_user_lock_p __kmp_lookup_user_lock(void **user_lock,
- char const *func);
+extern kmp_user_lock_p __kmp_user_lock_allocate( void **user_lock, kmp_int32 gtid, kmp_lock_flags_t flags );
+extern void __kmp_user_lock_free( void **user_lock, kmp_int32 gtid, kmp_user_lock_p lck );
+extern kmp_user_lock_p __kmp_lookup_user_lock( void **user_lock, char const *func );
extern void __kmp_cleanup_user_locks();
-#define KMP_CHECK_USER_LOCK_INIT() \
- { \
- if (!TCR_4(__kmp_init_user_locks)) { \
- __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); \
- if (!TCR_4(__kmp_init_user_locks)) { \
- TCW_4(__kmp_init_user_locks, TRUE); \
- } \
- __kmp_release_bootstrap_lock(&__kmp_initz_lock); \
- } \
- }
+#define KMP_CHECK_USER_LOCK_INIT() \
+ { \
+ if ( ! TCR_4( __kmp_init_user_locks ) ) { \
+ __kmp_acquire_bootstrap_lock( &__kmp_initz_lock ); \
+ if ( ! TCR_4( __kmp_init_user_locks ) ) { \
+ TCW_4( __kmp_init_user_locks, TRUE ); \
+ } \
+ __kmp_release_bootstrap_lock( &__kmp_initz_lock ); \
+ } \
+ }
#endif // KMP_USE_DYNAMIC_LOCK
@@ -995,281 +1033,241 @@ extern void __kmp_cleanup_user_locks();
#undef KMP_GTID_DNE
#if KMP_USE_DYNAMIC_LOCK
-// KMP_USE_DYNAMIC_LOCK enables dynamic dispatch of lock functions without
-// breaking the current compatibility. Essential functionality of this new code
-// is dynamic dispatch, but it also implements (or enables implementation of)
-// hinted user lock and critical section which will be part of OMP 4.5 soon.
+
+//
+// KMP_USE_DYNAMIC_LOCK enables dynamic dispatch of lock functions without breaking the current
+// compatibility. Essential functionality of this new code is dynamic dispatch, but it also
+// implements (or enables implementation of) hinted user lock and critical section which will be
+// part of OMP 4.1 soon.
//
-// Lock type can be decided at creation time (i.e., lock initialization), and
-// subsequent lock function call on the created lock object requires type
-// extraction and call through jump table using the extracted type. This type
-// information is stored in two different ways depending on the size of the lock
-// object, and we differentiate lock types by this size requirement - direct and
-// indirect locks.
+// Lock type can be decided at creation time (i.e., lock initialization), and subsequent lock
+// function call on the created lock object requires type extraction and call through jump table
+// using the extracted type. This type information is stored in two different ways depending on
+// the size of the lock object, and we differentiate lock types by this size requirement - direct
+// and indirect locks.
//
// Direct locks:
-// A direct lock object fits into the space created by the compiler for an
-// omp_lock_t object, and TAS/Futex lock falls into this category. We use low
-// one byte of the lock object as the storage for the lock type, and appropriate
-// bit operation is required to access the data meaningful to the lock
-// algorithms. Also, to differentiate direct lock from indirect lock, 1 is
-// written to LSB of the lock object. The newly introduced "hle" lock is also a
-// direct lock.
+// A direct lock object fits into the space created by the compiler for an omp_lock_t object, and
+// TAS/Futex lock falls into this category. We use low one byte of the lock object as the storage
+// for the lock type, and appropriate bit operation is required to access the data meaningful to
+// the lock algorithms. Also, to differentiate direct lock from indirect lock, 1 is written to LSB
+// of the lock object. The newly introduced "hle" lock is also a direct lock.
//
// Indirect locks:
-// An indirect lock object requires more space than the compiler-generated
-// space, and it should be allocated from heap. Depending on the size of the
-// compiler-generated space for the lock (i.e., size of omp_lock_t), this
-// omp_lock_t object stores either the address of the heap-allocated indirect
-// lock (void * fits in the object) or an index to the indirect lock table entry
-// that holds the address. Ticket/Queuing/DRDPA/Adaptive lock falls into this
-// category, and the newly introduced "rtm" lock is also an indirect lock which
-// was implemented on top of the Queuing lock. When the omp_lock_t object holds
-// an index (not lock address), 0 is written to LSB to differentiate the lock
-// from a direct lock, and the remaining part is the actual index to the
+// An indirect lock object requires more space than the compiler-generated space, and it should be
+// allocated from heap. Depending on the size of the compiler-generated space for the lock (i.e.,
+// size of omp_lock_t), this omp_lock_t object stores either the address of the heap-allocated
+// indirect lock (void * fits in the object) or an index to the indirect lock table entry that
+// holds the address. Ticket/Queuing/DRDPA/Adaptive lock falls into this category, and the newly
+// introduced "rtm" lock is also an indirect lock which was implemented on top of the Queuing lock.
+// When the omp_lock_t object holds an index (not lock address), 0 is written to LSB to
+// differentiate the lock from a direct lock, and the remaining part is the actual index to the
// indirect lock table.
+//
#include <stdint.h> // for uintptr_t
// Shortcuts
-#define KMP_USE_INLINED_TAS \
- (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1
-#define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0
+#define KMP_USE_FUTEX (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
+#define KMP_USE_INLINED_TAS (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1
+#define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0
// List of lock definitions; all nested locks are indirect locks.
// hle lock is xchg lock prefixed with XACQUIRE/XRELEASE.
// All nested locks are indirect lock types.
#if KMP_USE_TSX
-#if KMP_USE_FUTEX
-#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) m(rtm_spin, a)
-#define KMP_FOREACH_I_LOCK(m, a) \
- m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a) \
- m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
- m(nested_queuing, a) m(nested_drdpa, a)
-#else
-#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) m(rtm_spin, a)
-#define KMP_FOREACH_I_LOCK(m, a) \
- m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a) \
- m(nested_tas, a) m(nested_ticket, a) m(nested_queuing, a) \
- m(nested_drdpa, a)
-#endif // KMP_USE_FUTEX
-#define KMP_LAST_D_LOCK lockseq_rtm_spin
+# if KMP_USE_FUTEX
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \
+ m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
+ m(nested_queuing, a) m(nested_drdpa, a)
+# else
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a)
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a) \
+ m(nested_tas, a) m(nested_ticket, a) \
+ m(nested_queuing, a) m(nested_drdpa, a)
+# endif // KMP_USE_FUTEX
+# define KMP_LAST_D_LOCK lockseq_hle
#else
-#if KMP_USE_FUTEX
-#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
-#define KMP_FOREACH_I_LOCK(m, a) \
- m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_futex, a) \
- m(nested_ticket, a) m(nested_queuing, a) m(nested_drdpa, a)
-#define KMP_LAST_D_LOCK lockseq_futex
-#else
-#define KMP_FOREACH_D_LOCK(m, a) m(tas, a)
-#define KMP_FOREACH_I_LOCK(m, a) \
- m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_ticket, a) \
- m(nested_queuing, a) m(nested_drdpa, a)
-#define KMP_LAST_D_LOCK lockseq_tas
-#endif // KMP_USE_FUTEX
+# if KMP_USE_FUTEX
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
+ m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
+ m(nested_queuing, a) m(nested_drdpa, a)
+# define KMP_LAST_D_LOCK lockseq_futex
+# else
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a)
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
+ m(nested_tas, a) m(nested_ticket, a) \
+ m(nested_queuing, a) m(nested_drdpa, a)
+# define KMP_LAST_D_LOCK lockseq_tas
+# endif // KMP_USE_FUTEX
#endif // KMP_USE_TSX
// Information used in dynamic dispatch
-#define KMP_LOCK_SHIFT \
- 8 // number of low bits to be used as tag for direct locks
+#define KMP_LOCK_SHIFT 8 // number of low bits to be used as tag for direct locks
#define KMP_FIRST_D_LOCK lockseq_tas
#define KMP_FIRST_I_LOCK lockseq_ticket
-#define KMP_LAST_I_LOCK lockseq_nested_drdpa
-#define KMP_NUM_I_LOCKS \
- (locktag_nested_drdpa + 1) // number of indirect lock types
+#define KMP_LAST_I_LOCK lockseq_nested_drdpa
+#define KMP_NUM_I_LOCKS (locktag_nested_drdpa+1) // number of indirect lock types
// Base type for dynamic locks.
typedef kmp_uint32 kmp_dyna_lock_t;
-// Lock sequence that enumerates all lock kinds. Always make this enumeration
-// consistent with kmp_lockseq_t in the include directory.
+// Lock sequence that enumerates all lock kinds.
+// Always make this enumeration consistent with kmp_lockseq_t in the include directory.
typedef enum {
- lockseq_indirect = 0,
-#define expand_seq(l, a) lockseq_##l,
- KMP_FOREACH_D_LOCK(expand_seq, 0) KMP_FOREACH_I_LOCK(expand_seq, 0)
+ lockseq_indirect = 0,
+#define expand_seq(l,a) lockseq_##l,
+ KMP_FOREACH_D_LOCK(expand_seq, 0)
+ KMP_FOREACH_I_LOCK(expand_seq, 0)
#undef expand_seq
} kmp_dyna_lockseq_t;
// Enumerates indirect lock tags.
typedef enum {
-#define expand_tag(l, a) locktag_##l,
- KMP_FOREACH_I_LOCK(expand_tag, 0)
+#define expand_tag(l,a) locktag_##l,
+ KMP_FOREACH_I_LOCK(expand_tag, 0)
#undef expand_tag
} kmp_indirect_locktag_t;
// Utility macros that extract information from lock sequences.
-#define KMP_IS_D_LOCK(seq) \
- ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK)
-#define KMP_IS_I_LOCK(seq) \
- ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK)
-#define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq)-KMP_FIRST_I_LOCK)
-#define KMP_GET_D_TAG(seq) ((seq) << 1 | 1)
+#define KMP_IS_D_LOCK(seq) ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK)
+#define KMP_IS_I_LOCK(seq) ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK)
+#define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq) - KMP_FIRST_I_LOCK)
+#define KMP_GET_D_TAG(seq) ((seq)<<1 | 1)
// Enumerates direct lock tags starting from indirect tag.
typedef enum {
-#define expand_tag(l, a) locktag_##l = KMP_GET_D_TAG(lockseq_##l),
- KMP_FOREACH_D_LOCK(expand_tag, 0)
+#define expand_tag(l,a) locktag_##l = KMP_GET_D_TAG(lockseq_##l),
+ KMP_FOREACH_D_LOCK(expand_tag, 0)
#undef expand_tag
} kmp_direct_locktag_t;
// Indirect lock type
typedef struct {
- kmp_user_lock_p lock;
- kmp_indirect_locktag_t type;
+ kmp_user_lock_p lock;
+ kmp_indirect_locktag_t type;
} kmp_indirect_lock_t;
-// Function tables for direct locks. Set/unset/test differentiate functions
-// with/without consistency checking.
+// Function tables for direct locks. Set/unset/test differentiate functions with/without consistency checking.
extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
-extern void (**__kmp_direct_destroy)(kmp_dyna_lock_t *);
-extern int (**__kmp_direct_set)(kmp_dyna_lock_t *, kmp_int32);
-extern int (**__kmp_direct_unset)(kmp_dyna_lock_t *, kmp_int32);
-extern int (**__kmp_direct_test)(kmp_dyna_lock_t *, kmp_int32);
+extern void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *);
+extern void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32);
+extern int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32);
+extern int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32);
-// Function tables for indirect locks. Set/unset/test differentiate functions
-// with/without consistency checking.
+// Function tables for indirect locks. Set/unset/test differentiate functions with/withuot consistency checking.
extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
-extern void (**__kmp_indirect_destroy)(kmp_user_lock_p);
-extern int (**__kmp_indirect_set)(kmp_user_lock_p, kmp_int32);
-extern int (**__kmp_indirect_unset)(kmp_user_lock_p, kmp_int32);
-extern int (**__kmp_indirect_test)(kmp_user_lock_p, kmp_int32);
+extern void (*__kmp_indirect_destroy[])(kmp_user_lock_p);
+extern void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32);
+extern int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32);
+extern int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32);
// Extracts direct lock tag from a user lock pointer
-#define KMP_EXTRACT_D_TAG(l) \
- (*((kmp_dyna_lock_t *)(l)) & ((1 << KMP_LOCK_SHIFT) - 1) & \
- -(*((kmp_dyna_lock_t *)(l)) & 1))
+#define KMP_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & ((1<<KMP_LOCK_SHIFT)-1) & -(*((kmp_dyna_lock_t *)(l)) & 1))
// Extracts indirect lock index from a user lock pointer
#define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1)
-// Returns function pointer to the direct lock function with l (kmp_dyna_lock_t
-// *) and op (operation type).
+// Returns function pointer to the direct lock function with l (kmp_dyna_lock_t *) and op (operation type).
#define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)]
-// Returns function pointer to the indirect lock function with l
-// (kmp_indirect_lock_t *) and op (operation type).
-#define KMP_I_LOCK_FUNC(l, op) \
- __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type]
+// Returns function pointer to the indirect lock function with l (kmp_indirect_lock_t *) and op (operation type).
+#define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type]
// Initializes a direct lock with the given lock pointer and lock sequence.
-#define KMP_INIT_D_LOCK(l, seq) \
- __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
+#define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
// Initializes an indirect lock with the given lock pointer and lock sequence.
-#define KMP_INIT_I_LOCK(l, seq) \
- __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq)
+#define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq)
// Returns "free" lock value for the given lock type.
-#define KMP_LOCK_FREE(type) (locktag_##type)
+#define KMP_LOCK_FREE(type) (locktag_##type)
// Returns "busy" lock value for the given lock teyp.
-#define KMP_LOCK_BUSY(v, type) ((v) << KMP_LOCK_SHIFT | locktag_##type)
+#define KMP_LOCK_BUSY(v, type) ((v)<<KMP_LOCK_SHIFT | locktag_##type)
// Returns lock value after removing (shifting) lock tag.
-#define KMP_LOCK_STRIP(v) ((v) >> KMP_LOCK_SHIFT)
+#define KMP_LOCK_STRIP(v) ((v)>>KMP_LOCK_SHIFT)
-// Initializes global states and data structures for managing dynamic user
-// locks.
+// Initializes global states and data structures for managing dynamic user locks.
extern void __kmp_init_dynamic_user_locks();
// Allocates and returns an indirect lock with the given indirect lock tag.
-extern kmp_indirect_lock_t *
-__kmp_allocate_indirect_lock(void **, kmp_int32, kmp_indirect_locktag_t);
+extern kmp_indirect_lock_t * __kmp_allocate_indirect_lock(void **, kmp_int32, kmp_indirect_locktag_t);
// Cleans up global states and data structures for managing dynamic user locks.
extern void __kmp_cleanup_indirect_user_locks();
-// Default user lock sequence when not using hinted locks.
+// Default user lock sequence when not using hinted locks.
extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
// Jump table for "set lock location", available only for indirect locks.
-extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
- const ident_t *);
-#define KMP_SET_I_LOCK_LOCATION(lck, loc) \
- { \
- if (__kmp_indirect_set_location[(lck)->type] != NULL) \
- __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \
- }
+extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p, const ident_t *);
+#define KMP_SET_I_LOCK_LOCATION(lck, loc) { \
+ if (__kmp_indirect_set_location[(lck)->type] != NULL) \
+ __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \
+}
// Jump table for "set lock flags", available only for indirect locks.
-extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
- kmp_lock_flags_t);
-#define KMP_SET_I_LOCK_FLAGS(lck, flag) \
- { \
- if (__kmp_indirect_set_flags[(lck)->type] != NULL) \
- __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \
- }
+extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_lock_flags_t);
+#define KMP_SET_I_LOCK_FLAGS(lck, flag) { \
+ if (__kmp_indirect_set_flags[(lck)->type] != NULL) \
+ __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \
+}
// Jump table for "get lock location", available only for indirect locks.
-extern const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
- kmp_user_lock_p);
-#define KMP_GET_I_LOCK_LOCATION(lck) \
- (__kmp_indirect_get_location[(lck)->type] != NULL \
- ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \
- : NULL)
+extern const ident_t * (*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p);
+#define KMP_GET_I_LOCK_LOCATION(lck) ( __kmp_indirect_get_location[(lck)->type] != NULL \
+ ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \
+ : NULL )
// Jump table for "get lock flags", available only for indirect locks.
-extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
- kmp_user_lock_p);
-#define KMP_GET_I_LOCK_FLAGS(lck) \
- (__kmp_indirect_get_flags[(lck)->type] != NULL \
- ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \
- : NULL)
+extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p);
+#define KMP_GET_I_LOCK_FLAGS(lck) ( __kmp_indirect_get_flags[(lck)->type] != NULL \
+ ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \
+ : NULL )
-#define KMP_I_LOCK_CHUNK \
- 1024 // number of kmp_indirect_lock_t objects to be allocated together
+#define KMP_I_LOCK_CHUNK 1024 // number of kmp_indirect_lock_t objects to be allocated together
// Lock table for indirect locks.
typedef struct kmp_indirect_lock_table {
- kmp_indirect_lock_t **table; // blocks of indirect locks allocated
- kmp_lock_index_t size; // size of the indirect lock table
- kmp_lock_index_t next; // index to the next lock to be allocated
+ kmp_indirect_lock_t **table; // blocks of indirect locks allocated
+ kmp_lock_index_t size; // size of the indirect lock table
+ kmp_lock_index_t next; // index to the next lock to be allocated
} kmp_indirect_lock_table_t;
extern kmp_indirect_lock_table_t __kmp_i_lock_table;
// Returns the indirect lock associated with the given index.
-#define KMP_GET_I_LOCK(index) \
- (*(__kmp_i_lock_table.table + (index) / KMP_I_LOCK_CHUNK) + \
- (index) % KMP_I_LOCK_CHUNK)
+#define KMP_GET_I_LOCK(index) (*(__kmp_i_lock_table.table + (index)/KMP_I_LOCK_CHUNK) + (index)%KMP_I_LOCK_CHUNK)
// Number of locks in a lock block, which is fixed to "1" now.
-// TODO: No lock block implementation now. If we do support, we need to manage
-// lock block data structure for each indirect lock type.
+// TODO: No lock block implementation now. If we do support, we need to manage lock block data
+// structure for each indirect lock type.
extern int __kmp_num_locks_in_block;
// Fast lock table lookup without consistency checking
-#define KMP_LOOKUP_I_LOCK(l) \
- ((OMP_LOCK_T_SIZE < sizeof(void *)) ? KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(l)) \
- : *((kmp_indirect_lock_t **)(l)))
+#define KMP_LOOKUP_I_LOCK(l) ( (OMP_LOCK_T_SIZE < sizeof(void *)) \
+ ? KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(l)) \
+ : *((kmp_indirect_lock_t **)(l)) )
-// Used once in kmp_error.cpp
-extern kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
+// Used once in kmp_error.c
+extern kmp_int32
+__kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
#else // KMP_USE_DYNAMIC_LOCK
-#define KMP_LOCK_BUSY(v, type) (v)
-#define KMP_LOCK_FREE(type) 0
-#define KMP_LOCK_STRIP(v) (v)
+# define KMP_LOCK_BUSY(v, type) (v)
+# define KMP_LOCK_FREE(type) 0
+# define KMP_LOCK_STRIP(v) (v)
#endif // KMP_USE_DYNAMIC_LOCK
-// data structure for using backoff within spin locks.
-typedef struct {
- kmp_uint32 step; // current step
- kmp_uint32 max_backoff; // upper bound of outer delay loop
- kmp_uint32 min_tick; // size of inner delay loop in ticks (machine-dependent)
-} kmp_backoff_t;
-
-// Runtime's default backoff parameters
-extern kmp_backoff_t __kmp_spin_backoff_params;
-
-// Backoff function
-extern void __kmp_spin_backoff(kmp_backoff_t *);
-
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
#endif /* KMP_LOCK_H */
+