diff options
author | robot-contrib <robot-contrib@yandex-team.com> | 2023-04-29 13:37:01 +0300 |
---|---|---|
committer | robot-contrib <robot-contrib@yandex-team.com> | 2023-04-29 13:37:01 +0300 |
commit | 9a5be7417481c2ade2dcc72952a3320600256084 (patch) | |
tree | 68135983be6a0c7d2b7543c00f2e21138756fcee | |
parent | cfb5240e49217e65df1b97a1cacb804c077b2ba7 (diff) | |
download | ydb-9a5be7417481c2ade2dcc72952a3320600256084.tar.gz |
Update contrib/restricted/boost/atomic to 1.82.0
5 files changed, 144 insertions, 107 deletions
diff --git a/contrib/restricted/boost/atomic/include/boost/atomic/atomic.hpp b/contrib/restricted/boost/atomic/include/boost/atomic/atomic.hpp index 8bd075a9a7..126a0431bd 100644 --- a/contrib/restricted/boost/atomic/include/boost/atomic/atomic.hpp +++ b/contrib/restricted/boost/atomic/include/boost/atomic/atomic.hpp @@ -46,9 +46,6 @@ private: public: typedef typename base_type::value_type value_type; - // Deprecated, use value_type instead - BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED - typedef typename base_type::storage_type storage_type; BOOST_STATIC_ASSERT_MSG(sizeof(value_type) > 0u, "boost::atomic<T> requires T to be a complete type"); #if !defined(BOOST_ATOMIC_DETAIL_NO_CXX11_IS_TRIVIALLY_COPYABLE) @@ -81,16 +78,6 @@ public: return this->load(); } - // Deprecated, use value() instead - BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED - BOOST_FORCEINLINE typename base_type::storage_type& storage() BOOST_NOEXCEPT { return base_type::storage(); } - BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED - BOOST_FORCEINLINE typename base_type::storage_type volatile& storage() volatile BOOST_NOEXCEPT { return base_type::storage(); } - BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED - BOOST_FORCEINLINE typename base_type::storage_type const& storage() const BOOST_NOEXCEPT { return base_type::storage(); } - BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED - BOOST_FORCEINLINE typename base_type::storage_type const volatile& storage() const volatile BOOST_NOEXCEPT { return base_type::storage(); } - BOOST_DELETED_FUNCTION(atomic(atomic const&)) BOOST_DELETED_FUNCTION(atomic& operator= (atomic const&)) BOOST_DELETED_FUNCTION(atomic& operator= (atomic const&) volatile) diff --git a/contrib/restricted/boost/atomic/include/boost/atomic/detail/config.hpp b/contrib/restricted/boost/atomic/include/boost/atomic/detail/config.hpp index 1e9f1bb1a9..185f0cc240 100644 --- a/contrib/restricted/boost/atomic/include/boost/atomic/detail/config.hpp +++ b/contrib/restricted/boost/atomic/include/boost/atomic/detail/config.hpp @@ -140,53 +140,4 @@ #define BOOST_ATOMIC_DETAIL_INT_FP_ENDIAN_MATCH #endif -// Deprecated symbols markup -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) && defined(_MSC_VER) -#if (_MSC_VER) >= 1400 -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) __declspec(deprecated(msg)) -#else -// MSVC 7.1 only supports the attribute without a message -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) __declspec(deprecated) -#endif -#endif - -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) && defined(__has_extension) -#if __has_extension(attribute_deprecated_with_message) -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) __attribute__((deprecated(msg))) -#endif -#endif - -// gcc since 4.5 supports deprecated attribute with a message; older versions support the attribute without a message. -// Oracle Studio 12.4 supports deprecated attribute with a message; this is the first release that supports the attribute. -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) && (\ - (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__) >= 405) ||\ - (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5130)) -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) __attribute__((deprecated(msg))) -#endif - -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) && __cplusplus >= 201402 -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) [[deprecated(msg)]] -#endif - -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) && defined(__GNUC__) -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) __attribute__((deprecated)) -#endif - -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) && defined(__has_attribute) -#if __has_attribute(deprecated) -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) __attribute__((deprecated)) -#endif -#endif - -#if !defined(BOOST_ATOMIC_DETAIL_DEPRECATED) -#define BOOST_ATOMIC_DETAIL_DEPRECATED(msg) -#endif - -// In Boost.Atomic 1.73 we deprecated atomic<>::storage() accessor in favor of atomic<>::value(). In future releases storage() will be removed. -#if !defined(BOOST_ATOMIC_SILENCE_STORAGE_DEPRECATION) -#define BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED BOOST_ATOMIC_DETAIL_DEPRECATED("Boost.Atomic 1.73 has deprecated atomic<>::storage() in favor of atomic<>::value() and atomic<>::storage_type in favor of atomic<>::value_type. You can define BOOST_ATOMIC_SILENCE_STORAGE_DEPRECATION to disable this warning.") -#else -#define BOOST_ATOMIC_DETAIL_STORAGE_DEPRECATED -#endif - #endif // BOOST_ATOMIC_DETAIL_CONFIG_HPP_INCLUDED_ diff --git a/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_gcc_x86.hpp b/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_gcc_x86.hpp index 7f41ff843d..b0e3936e3a 100644 --- a/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_gcc_x86.hpp +++ b/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_gcc_x86.hpp @@ -415,6 +415,11 @@ struct gcc_dcas_x86 { typedef typename storage_traits< 8u >::type storage_type; typedef uint32_t BOOST_ATOMIC_DETAIL_MAY_ALIAS aliasing_uint32_t; +#if defined(__SSE2__) + typedef uint32_t xmm_t __attribute__((__vector_size__(16))); +#elif defined(__SSE__) + typedef float xmm_t __attribute__((__vector_size__(16))); +#endif static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 8u; static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 8u; @@ -423,31 +428,32 @@ struct gcc_dcas_x86 static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true; static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true; - static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT + static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT { - if (BOOST_LIKELY((((uintptr_t)&storage) & 0x00000007) == 0u)) + if (BOOST_LIKELY(order != memory_order_seq_cst && (((uintptr_t)&storage) & 7u) == 0u)) { #if defined(__SSE__) - typedef float xmm_t __attribute__((__vector_size__(16))); - xmm_t xmm_scratch; +#if defined(__SSE2__) + xmm_t value = { static_cast< uint32_t >(v), static_cast< uint32_t >(v >> 32u), 0u, 0u }; +#else + xmm_t value; + BOOST_ATOMIC_DETAIL_MEMSET(&value, 0, sizeof(value)); + BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(v)); +#endif __asm__ __volatile__ ( #if defined(__AVX__) - "vmovq %[value], %[xmm_scratch]\n\t" - "vmovq %[xmm_scratch], %[storage]\n\t" + "vmovq %[value], %[storage]\n\t" #elif defined(__SSE2__) - "movq %[value], %[xmm_scratch]\n\t" - "movq %[xmm_scratch], %[storage]\n\t" + "movq %[value], %[storage]\n\t" #else - "xorps %[xmm_scratch], %[xmm_scratch]\n\t" - "movlps %[value], %[xmm_scratch]\n\t" - "movlps %[xmm_scratch], %[storage]\n\t" + "movlps %[value], %[storage]\n\t" #endif - : [storage] "=m" (storage), [xmm_scratch] "=x" (xmm_scratch) - : [value] "m" (v) + : [storage] "=m" (storage) + : [value] "x" (value) : "memory" ); -#else +#else // defined(__SSE__) __asm__ __volatile__ ( "fildll %[value]\n\t" @@ -456,7 +462,7 @@ struct gcc_dcas_x86 : [value] "m" (v) : "memory" ); -#endif +#endif // defined(__SSE__) } else { @@ -472,7 +478,7 @@ struct gcc_dcas_x86 "jne 1b\n\t" "xchgl %%ebx, %%esi\n\t" : - : "a" ((uint32_t)v), "c" ((uint32_t)(v >> 32)), [dest] "D" (&storage) + : "a" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "D" (&storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "edx", "memory" ); #else // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX) @@ -484,7 +490,7 @@ struct gcc_dcas_x86 "1: lock; cmpxchg8b %[dest_lo]\n\t" "jne 1b\n\t" : [dest_lo] "=m" (storage), [dest_hi] "=m" (reinterpret_cast< volatile aliasing_uint32_t* >(&storage)[1]) - : [value_lo] "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32)) + : [value_lo] "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "eax", "edx", "memory" ); #endif // defined(BOOST_ATOMIC_DETAIL_X86_ASM_PRESERVE_EBX) @@ -495,29 +501,31 @@ struct gcc_dcas_x86 { storage_type value; - if (BOOST_LIKELY((((uintptr_t)&storage) & 0x00000007) == 0u)) + if (BOOST_LIKELY((((uintptr_t)&storage) & 7u) == 0u)) { #if defined(__SSE__) - typedef float xmm_t __attribute__((__vector_size__(16))); - xmm_t xmm_scratch; + xmm_t v; __asm__ __volatile__ ( #if defined(__AVX__) - "vmovq %[storage], %[xmm_scratch]\n\t" - "vmovq %[xmm_scratch], %[value]\n\t" + "vmovq %[storage], %[value]\n\t" #elif defined(__SSE2__) - "movq %[storage], %[xmm_scratch]\n\t" - "movq %[xmm_scratch], %[value]\n\t" + "movq %[storage], %[value]\n\t" #else - "xorps %[xmm_scratch], %[xmm_scratch]\n\t" - "movlps %[storage], %[xmm_scratch]\n\t" - "movlps %[xmm_scratch], %[value]\n\t" + "xorps %[value], %[value]\n\t" + "movlps %[storage], %[value]\n\t" #endif - : [value] "=m" (value), [xmm_scratch] "=x" (xmm_scratch) + : [value] "=x" (v) : [storage] "m" (storage) : "memory" ); +#if defined(__SSE2__) && (!defined(BOOST_GCC) || BOOST_GCC >= 40800) + // gcc prior to 4.8 don't support subscript operator for vector types + value = static_cast< storage_type >(v[0]) | (static_cast< storage_type >(v[1]) << 32u); #else + BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(value)); +#endif +#else // defined(__SSE__) __asm__ __volatile__ ( "fildll %[storage]\n\t" @@ -526,7 +534,7 @@ struct gcc_dcas_x86 : [storage] "m" (storage) : "memory" ); -#endif +#endif // defined(__SSE__) } else { @@ -548,7 +556,7 @@ struct gcc_dcas_x86 : [storage] "m" (storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); - BOOST_ATOMIC_DETAIL_MEMCPY(&value, value_bits, sizeof(value)); + value = static_cast< storage_type >(value_bits[0]) | (static_cast< storage_type >(value_bits[1]) << 32u); #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS) @@ -591,7 +599,7 @@ struct gcc_dcas_x86 "lock; cmpxchg8b (%[dest])\n\t" "xchgl %%ebx, %%esi\n\t" : "+A" (expected), [success] "=@ccz" (success) - : "S" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32)), [dest] "D" (&storage) + : "S" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u)), [dest] "D" (&storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) @@ -602,7 +610,7 @@ struct gcc_dcas_x86 "xchgl %%ebx, %%esi\n\t" "sete %[success]\n\t" : "+A" (expected), [success] "=qm" (success) - : "S" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32)), [dest] "D" (&storage) + : "S" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u)), [dest] "D" (&storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) @@ -618,7 +626,7 @@ struct gcc_dcas_x86 ( "lock; cmpxchg8b %[dest]\n\t" : "+A" (expected), [dest] "+m" (storage), [success] "=@ccz" (success) - : "b" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32)) + : "b" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u)) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); #else // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) @@ -627,7 +635,7 @@ struct gcc_dcas_x86 "lock; cmpxchg8b %[dest]\n\t" "sete %[success]\n\t" : "+A" (expected), [dest] "+m" (storage), [success] "=qm" (success) - : "b" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32)) + : "b" ((uint32_t)desired), "c" ((uint32_t)(desired >> 32u)) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); #endif // defined(BOOST_ATOMIC_DETAIL_ASM_HAS_FLAG_OUTPUTS) @@ -659,7 +667,7 @@ struct gcc_dcas_x86 "jne 1b\n\t" "xchgl %%ebx, %%esi\n\t" : "=a" (old_bits[0]), "=d" (old_bits[1]) - : "S" ((uint32_t)v), "c" ((uint32_t)(v >> 32)), [dest] "D" (&storage) + : "S" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "D" (&storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); @@ -680,7 +688,7 @@ struct gcc_dcas_x86 "jne 1b\n\t" "xchgl %%ebx, %%esi\n\t" : "=A" (old_value) - : "S" ((uint32_t)v), "c" ((uint32_t)(v >> 32)), [dest] "D" (&storage) + : "S" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "D" (&storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); return old_value; @@ -699,7 +707,7 @@ struct gcc_dcas_x86 "1: lock; cmpxchg8b (%[dest])\n\t" "jne 1b\n\t" : "=&a" (old_bits[0]), "=&d" (old_bits[1]) - : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32)), [dest] "DS" (&storage) + : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)), [dest] "DS" (&storage) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); @@ -718,7 +726,7 @@ struct gcc_dcas_x86 "1: lock; cmpxchg8b %[dest_lo]\n\t" "jne 1b\n\t" : "=&a" (old_bits[0]), "=&d" (old_bits[1]), [dest_lo] "+m" (storage), [dest_hi] "+m" (reinterpret_cast< volatile aliasing_uint32_t* >(&storage)[1]) - : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32)) + : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); @@ -737,7 +745,7 @@ struct gcc_dcas_x86 "1: lock; cmpxchg8b %[dest_lo]\n\t" "jne 1b\n\t" : "=&A" (old_value), [dest_lo] "+m" (storage), [dest_hi] "+m" (reinterpret_cast< volatile aliasing_uint32_t* >(&storage)[1]) - : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32)) + : "b" ((uint32_t)v), "c" ((uint32_t)(v >> 32u)) : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); return old_value; @@ -860,6 +868,9 @@ struct gcc_dcas_x86_64 { typedef typename storage_traits< 16u >::type storage_type; typedef uint64_t BOOST_ATOMIC_DETAIL_MAY_ALIAS aliasing_uint64_t; +#if defined(__AVX__) + typedef uint64_t __attribute__((__vector_size__(16))) xmm_t; +#endif static BOOST_CONSTEXPR_OR_CONST std::size_t storage_size = 16u; static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 16u; @@ -868,8 +879,31 @@ struct gcc_dcas_x86_64 static BOOST_CONSTEXPR_OR_CONST bool full_cas_based = true; static BOOST_CONSTEXPR_OR_CONST bool is_always_lock_free = true; - static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT + static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT { +#if defined(__AVX__) + if (BOOST_LIKELY(order != memory_order_seq_cst && (((uintptr_t)&storage) & 15u) == 0u)) + { + // According to SDM Volume 3, 8.1.1 Guaranteed Atomic Operations, processors supporting AVX guarantee + // aligned vector moves to be atomic. +#if defined(BOOST_HAS_INT128) + xmm_t value = { static_cast< uint64_t >(v), static_cast< uint64_t >(v >> 64u) }; +#else + xmm_t value; + BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(v)); +#endif + __asm__ __volatile__ + ( + "vmovdqa %[value], %[storage]\n\t" + : [storage] "=m" (storage) + : [value] "x" (value) + : "memory" + ); + + return; + } +#endif // defined(__AVX__) + __asm__ __volatile__ ( "movq %[dest_lo], %%rax\n\t" @@ -885,6 +919,31 @@ struct gcc_dcas_x86_64 static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT { +#if defined(__AVX__) + if (BOOST_LIKELY((((uintptr_t)&storage) & 15u) == 0u)) + { + // According to SDM Volume 3, 8.1.1 Guaranteed Atomic Operations, processors supporting AVX guarantee + // aligned vector moves to be atomic. + xmm_t v; + __asm__ __volatile__ + ( + "vmovdqa %[storage], %[value]\n\t" + : [value] "=x" (v) + : [storage] "m" (storage) + : "memory" + ); + +#if defined(BOOST_HAS_INT128) && (!defined(BOOST_GCC) || BOOST_GCC >= 40800) + // gcc prior to 4.8 don't support subscript operator for vector types + storage_type value = static_cast< storage_type >(v[0]) | (static_cast< storage_type >(v[1]) << 64u); +#else + storage_type value; + BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(v)); +#endif + return value; + } +#endif // defined(__AVX__) + // Note that despite const qualification cmpxchg16b below may issue a store to the storage. The storage value // will not change, but this prevents the storage to reside in read-only memory. @@ -904,8 +963,12 @@ struct gcc_dcas_x86_64 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); +#if defined(BOOST_HAS_INT128) + storage_type value = static_cast< storage_type >(value_bits[0]) | (static_cast< storage_type >(value_bits[1]) << 64u); +#else storage_type value; BOOST_ATOMIC_DETAIL_MEMCPY(&value, value_bits, sizeof(value)); +#endif return value; #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS) @@ -1004,8 +1067,12 @@ struct gcc_dcas_x86_64 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory" ); +#if defined(BOOST_HAS_INT128) + storage_type old_value = static_cast< storage_type >(old_bits[0]) | (static_cast< storage_type >(old_bits[1]) << 64u); +#else storage_type old_value; BOOST_ATOMIC_DETAIL_MEMCPY(&old_value, old_bits, sizeof(old_value)); +#endif return old_value; #else // defined(BOOST_ATOMIC_DETAIL_X86_NO_ASM_AX_DX_PAIRS) storage_type old_value; diff --git a/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_msvc_x86.hpp b/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_msvc_x86.hpp index 29c9afb19a..197512a1ec 100644 --- a/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_msvc_x86.hpp +++ b/contrib/restricted/boost/atomic/include/boost/atomic/detail/core_arch_ops_msvc_x86.hpp @@ -31,6 +31,10 @@ #include <boost/atomic/detail/cas_based_exchange.hpp> #include <boost/atomic/detail/core_ops_cas_based.hpp> #endif +#if defined(BOOST_ATOMIC_DETAIL_X86_HAS_CMPXCHG16B) && defined(__AVX__) +#include <emmintrin.h> +#include <boost/atomic/detail/string_ops.hpp> +#endif #include <boost/atomic/detail/ops_msvc_common.hpp> #if !defined(_M_IX86) && !(defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE8) && defined(BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE16)) #include <boost/atomic/detail/extending_cas_based_arithmetic.hpp> @@ -587,12 +591,12 @@ struct msvc_dcas_x86 // Luckily, the memory is almost always 8-byte aligned in our case because atomic<> uses 64 bit native types for storage and dynamic memory allocations // have at least 8 byte alignment. The only unfortunate case is when atomic is placed on the stack and it is not 8-byte aligned (like on 32 bit Windows). - static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT + static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT { BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); storage_type volatile* p = &storage; - if (((uintptr_t)p & 0x00000007) == 0) + if (BOOST_LIKELY(order != memory_order_seq_cst && ((uintptr_t)p & 7u) == 0u)) { #if defined(_M_IX86_FP) && _M_IX86_FP >= 2 #if defined(__AVX__) @@ -648,7 +652,7 @@ struct msvc_dcas_x86 storage_type const volatile* p = &storage; storage_type value; - if (((uintptr_t)p & 0x00000007) == 0) + if (BOOST_LIKELY(((uintptr_t)p & 7u) == 0u)) { #if defined(_M_IX86_FP) && _M_IX86_FP >= 2 #if defined(__AVX__) @@ -833,15 +837,38 @@ struct msvc_dcas_x86_64 static BOOST_CONSTEXPR_OR_CONST std::size_t storage_alignment = 16u; static BOOST_CONSTEXPR_OR_CONST bool is_signed = Signed; - static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order) BOOST_NOEXCEPT + static BOOST_FORCEINLINE void store(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT { +#if defined(__AVX__) + if (BOOST_LIKELY(order != memory_order_seq_cst && (((uintptr_t)&storage) & 15u) == 0u)) + { + BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); + __m128i value; + BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(value)); + _mm_store_si128(const_cast< __m128i* >(reinterpret_cast< volatile __m128i* >(&storage)), value); + BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); + return; + } +#endif // defined(__AVX__) + storage_type value = const_cast< storage_type& >(storage); while (!BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, v, &value)) {} } static BOOST_FORCEINLINE storage_type load(storage_type const volatile& storage, memory_order) BOOST_NOEXCEPT { - storage_type value = storage_type(); + storage_type value; +#if defined(__AVX__) + if (BOOST_LIKELY((((uintptr_t)&storage) & 15u) == 0u)) + { + __m128i v = _mm_load_si128(const_cast< const __m128i* >(reinterpret_cast< const volatile __m128i* >(&storage))); + BOOST_ATOMIC_DETAIL_MEMCPY(&value, &v, sizeof(value)); + BOOST_ATOMIC_DETAIL_COMPILER_BARRIER(); + return value; + } +#endif // defined(__AVX__) + + value = storage_type(); BOOST_ATOMIC_INTERLOCKED_COMPARE_EXCHANGE128(&storage, value, &value); return value; } diff --git a/contrib/restricted/boost/atomic/src/find_address_sse2.cpp b/contrib/restricted/boost/atomic/src/find_address_sse2.cpp index 0739b33ca0..3e624dd5e9 100644 --- a/contrib/restricted/boost/atomic/src/find_address_sse2.cpp +++ b/contrib/restricted/boost/atomic/src/find_address_sse2.cpp @@ -52,7 +52,12 @@ BOOST_FORCEINLINE __m128i mm_pand_si128(__m128i mm1, __m128i mm2) // Sandy Bridge can execute 3 pand instructions per cycle, but only one andps. For this reason // we prefer to generate pand and not andps. #if defined(__GNUC__) +#if defined(__AVX__) + // Generate VEX-coded variant if the code is compiled for AVX and later. + __asm__("vpand %1, %0, %0\n\t" : "+x" (mm1) : "x" (mm2)); +#else __asm__("pand %1, %0\n\t" : "+x" (mm1) : "x" (mm2)); +#endif #else mm1 = _mm_and_si128(mm1, mm2); #endif |