diff options
| author | rumvadim <rumvadim@yandex-team.ru> | 2022-02-10 16:50:11 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:11 +0300 |
| commit | cf62db3a461da3c6fdd693fb4cfada80d16031f2 (patch) | |
| tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 /library | |
| parent | 03ed72c7c79029264440f45994a0b30e84919e80 (diff) | |
| download | ydb-cf62db3a461da3c6fdd693fb4cfada80d16031f2.tar.gz | |
Restoring authorship annotation for <rumvadim@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library')
| -rw-r--r-- | library/cpp/actors/core/actorsystem.h | 2 | ||||
| -rw-r--r-- | library/cpp/actors/testlib/test_runtime.cpp | 2 | ||||
| -rw-r--r-- | library/cpp/json/writer/json_value.cpp | 24 | ||||
| -rw-r--r-- | library/cpp/json/writer/json_value.h | 12 | ||||
| -rw-r--r-- | library/cpp/sse/powerpc.h | 468 | ||||
| -rw-r--r-- | library/cpp/sse/sse.h | 10 | ||||
| -rw-r--r-- | library/cpp/sse/sse2neon.h | 422 | ||||
| -rw-r--r-- | library/cpp/sse/ut/test.cpp | 1234 |
8 files changed, 1087 insertions, 1087 deletions
diff --git a/library/cpp/actors/core/actorsystem.h b/library/cpp/actors/core/actorsystem.h index 93e2f64d19..40499d7586 100644 --- a/library/cpp/actors/core/actorsystem.h +++ b/library/cpp/actors/core/actorsystem.h @@ -135,7 +135,7 @@ namespace NActors { public: virtual ~ISchedulerThread() { } - + virtual void Prepare(TActorSystem* actorSystem, volatile ui64* currentTimestamp, volatile ui64* currentMonotonic) = 0; virtual void PrepareSchedules(NSchedulerQueue::TReader** readers, ui32 scheduleReadersCount) = 0; virtual void PrepareStart() { /* empty */ } diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 18826ba6cd..6fa25b9965 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -1619,7 +1619,7 @@ namespace NActors { setup->Executors[2].Reset(new TIOExecutorPool(2, 1)); setup->Executors[3].Reset(new TBasicExecutorPool(3, 2, 20)); setup->Executors[4].Reset(new TBasicExecutorPool(4, 1, 20)); - setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(512, 100))); + setup->Scheduler.Reset(new TBasicSchedulerThread(TSchedulerConfig(512, 100))); } else { setup->ExecutorsCount = 1; setup->Scheduler.Reset(new TSchedulerThreadStub(this, node)); diff --git a/library/cpp/json/writer/json_value.cpp b/library/cpp/json/writer/json_value.cpp index 9e7d3d0c4d..c61e8d1dc4 100644 --- a/library/cpp/json/writer/json_value.cpp +++ b/library/cpp/json/writer/json_value.cpp @@ -249,12 +249,12 @@ namespace NJson { } TJsonValue& TJsonValue::Back() { - BackChecks(); - return Value.Array->back(); - } + BackChecks(); + return Value.Array->back(); + } - const TJsonValue& TJsonValue::Back() const { - BackChecks(); + const TJsonValue& TJsonValue::Back() const { + BackChecks(); return Value.Array->back(); } @@ -1089,13 +1089,13 @@ namespace NJson { return true; } - void TJsonValue::BackChecks() const { - if (Type != JSON_ARRAY) - ythrow TJsonException() << "Not an array"; - - if (Value.Array->empty()) - ythrow TJsonException() << "Get back on empty array"; - } + void TJsonValue::BackChecks() const { + if (Type != JSON_ARRAY) + ythrow TJsonException() << "Not an array"; + + if (Value.Array->empty()) + ythrow TJsonException() << "Get back on empty array"; + } } template <> diff --git a/library/cpp/json/writer/json_value.h b/library/cpp/json/writer/json_value.h index 965b9dc3c4..3f0f50bc4c 100644 --- a/library/cpp/json/writer/json_value.h +++ b/library/cpp/json/writer/json_value.h @@ -87,7 +87,7 @@ namespace NJson { TJsonValue& AppendValue(const TJsonValue& value); TJsonValue& AppendValue(TJsonValue&& value); TJsonValue& Back(); - const TJsonValue& Back() const; + const TJsonValue& Back() const; bool GetValueByPath(TStringBuf path, TJsonValue& result, char delimiter = '.') const; bool SetValueByPath(TStringBuf path, const TJsonValue& value, char delimiter = '.'); @@ -213,11 +213,11 @@ namespace NJson { TValueUnion Value; void DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback); void SwapWithUndefined(TJsonValue& output) noexcept; - - /** - @throw yexception if Back shouldn't be called on the object. - */ - void BackChecks() const; + + /** + @throw yexception if Back shouldn't be called on the object. + */ + void BackChecks() const; }; inline bool GetBoolean(const TJsonValue& jv, size_t index, bool* value) noexcept { diff --git a/library/cpp/sse/powerpc.h b/library/cpp/sse/powerpc.h index 49f6ef6306..82fc011fa5 100644 --- a/library/cpp/sse/powerpc.h +++ b/library/cpp/sse/powerpc.h @@ -1,23 +1,23 @@ -#pragma once - -/* - The header contains code which translates SSE intrinsics - to PowerPC AltiVec or software emulation. - - See also: https://www.ibm.com/developerworks/community/wikis/home?lang=en#!/wiki/W51a7ffcf4dfd_4b40_9d82_446ebc23c550/page/Intel%20SSE%20to%20PowerPC%20AltiVec%20migration -*/ -/* Author: Vadim Rumyantsev <rumvadim@yandex-team.ru> */ - -#if !defined(_ppc64_) -#error "This header is for PowerPC (ppc64) platform only." \ +#pragma once + +/* + The header contains code which translates SSE intrinsics + to PowerPC AltiVec or software emulation. + + See also: https://www.ibm.com/developerworks/community/wikis/home?lang=en#!/wiki/W51a7ffcf4dfd_4b40_9d82_446ebc23c550/page/Intel%20SSE%20to%20PowerPC%20AltiVec%20migration +*/ +/* Author: Vadim Rumyantsev <rumvadim@yandex-team.ru> */ + +#if !defined(_ppc64_) +#error "This header is for PowerPC (ppc64) platform only." \ "Include sse.h instead of including this header directly." -#endif - -#include <util/system/types.h> +#endif + +#include <util/system/types.h> #include <util/system/compiler.h> - -#include <altivec.h> - + +#include <altivec.h> + typedef __attribute__((__aligned__(8))) unsigned long long __m64; typedef __attribute__((__aligned__(16), __may_alias__)) vector float __m128; typedef __attribute__((__aligned__(16), __may_alias__)) vector unsigned char __m128i; @@ -47,49 +47,49 @@ enum _mm_hint #define _MM_SHUFFLE(a, b, c, d) ((signed char)(a * 64 + b * 16 + c * 4 + d)) -/// Functions that work with floats. - +/// Functions that work with floats. + Y_FORCE_INLINE __m128 _mm_setzero_ps() { return (__m128){0.0f, 0.0f, 0.0f, 0.0f}; -}; - +}; + Y_FORCE_INLINE __m128d _mm_setzero_pd() { return (__m128d)vec_splats((double)0); -} - +} + // bug in clang compiler until 7.0.0 inclusive, Y_NO_INLINE is vital/essential static Y_NO_INLINE __m128 _mm_set1_ps(float f) { return (vector float)f; -} - +} + Y_FORCE_INLINE __m128 _mm_set_ps1(float f) { - return _mm_set1_ps(f); -} - + return _mm_set1_ps(f); +} + Y_FORCE_INLINE __m128 _mm_set_ps(float v3, float v2, float v1, float v0) { return (__m128)(__v4sf){v0, v1, v2, v3}; -} - +} + Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { return (__m128d){d0, d1}; -} - +} + Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* p) { return vec_vsx_ld(0, p); -} - +} + Y_FORCE_INLINE __m128 _mm_load_ps(const float* p) { return (__m128)vec_ld(0, (vector float*)p); -} - +} + Y_FORCE_INLINE __m128 _mm_loadu_pd(const double* d) { return vec_vsx_ld(0, d); -} - +} + Y_FORCE_INLINE void _mm_storeu_ps(float* p, __m128 a) { *(__m128*)p = a; -} - +} + Y_FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) { return (__m128)vec_xor((__v4sf)a, (__v4sf)b); } @@ -100,77 +100,77 @@ Y_FORCE_INLINE __m128 _mm_xor_pd(__m128d a, __m128d b) { Y_FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) { return (__m128)((__v4sf)a + (__v4sf)b); -} - +} + Y_FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) { return (__m128d)((__v2df)a + (__v2df)b); -} - +} + Y_FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) { return (__m128)((__v4sf)a - (__v4sf)b); -} - +} + Y_FORCE_INLINE __m128d _mm_sub_pd(__m128d a, __m128d b) { return (__m128d)((__v2df)a - (__v2df)b); -} - +} + Y_FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) { return (__m128)((__v4sf)a * (__v4sf)b); -} - +} + Y_FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) { return (__m128d)((__v2df)a * (__v2df)b); -} - +} + Y_FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) { return (__m128)((__v4sf)a / (__v4sf)b); -} - +} + Y_FORCE_INLINE __m128d _mm_div_pd(__m128d a, __m128d b) { return (__m128d)((__v2df)a / (__v2df)b); -} - +} + Y_FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) { return ((__m128)vec_cmpeq((__v4sf)a, (__v4sf)b)); ; -} - +} + Y_FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) { return ((__m128)vec_cmpgt((__v4sf)a, (__v4sf)b)); -} - +} + Y_FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) { return (__m128)vec_max((vector float)a, (vector float)b); -} - +} + Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) { return (__m128i)vec_max((__v16qu)a, (__v16qu)b); } Y_FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) { return (__m128)vec_min((vector float)a, (vector float)b); -} - +} + Y_FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) { return ((__m128)vec_and((__v4sf)a, (__v4sf)b)); -} - +} + Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { return vec_and((__v2df)a, (__v2df)b); } Y_FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 a) { return vec_rsqrte(a); -} - +} + Y_FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 a) { __m128 a1, c; const vector unsigned int mask = {0xffffffff, 0, 0, 0}; a1 = vec_splat(a, 0); c = vec_rsqrte(a1); return (vec_sel((vector float)a, c, mask)); -} - +} + Y_FORCE_INLINE int _mm_movemask_ps(__m128 a) { __vector unsigned long long result; const __vector unsigned int perm_mask = @@ -190,16 +190,16 @@ Y_FORCE_INLINE int _mm_movemask_ps(__m128 a) { #elif __BIG_ENDIAN__ return result[0]; #endif -} - +} + Y_FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) { return ((__m128)vec_ctf((__v4si)a, 0)); -} - +} + Y_FORCE_INLINE float _mm_cvtss_f32(__m128 a) { return ((__v4sf)a)[0]; -} - +} + Y_FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 A, __m128 B) { __vector unsigned int a, b; __vector unsigned int c, d; @@ -211,32 +211,32 @@ Y_FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 A, __m128 B) { c = (__vector unsigned int)vec_cmpgt(a, float_exp_mask); d = (__vector unsigned int)vec_cmpgt(b, float_exp_mask); return ((__m128)vec_or(c, d)); -} - +} + Y_FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) { return ((__m128)vec_andc((__v4sf)b, (__v4sf)a)); -} - +} + Y_FORCE_INLINE __m128 _mm_or_ps(__m128 a, __m128 b) { return ((__m128)vec_or((__v4sf)a, (__v4sf)b)); -} - +} + Y_FORCE_INLINE void _mm_store_ss(float* p, __m128 a) { *p = ((__v4sf)a)[0]; -} - +} + Y_FORCE_INLINE void _mm_store_ps(float* p, __m128 a) { vec_st(a, 0, p); -} - +} + Y_FORCE_INLINE void _mm_storeu_pd(double* p, __m128d a) { *(__m128d*)p = a; -} - +} + Y_FORCE_INLINE void _mm_store_pd(double* p, __m128d a) { vec_st((vector unsigned char)a, 0, (vector unsigned char*)p); -} - +} + Y_FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, long shuff) { unsigned long element_selector_10 = shuff & 0x03; unsigned long element_selector_32 = (shuff >> 2) & 0x03; @@ -264,8 +264,8 @@ Y_FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, long shuff) { t[0] = permute_selectors[element_selector_76]; #endif return vec_perm((__v4sf)a, (__v4sf)b, (__vector unsigned char)t); -} - +} + Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, const int mask) { __vector double result; const int litmsk = mask & 0x3; @@ -288,10 +288,10 @@ Y_FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) { rounded = vec_rint((vector float)a); result = vec_cts(rounded, 0); return (__m128i)result; -} - -/// Functions that work with integers. - +} + +/// Functions that work with integers. + Y_FORCE_INLINE int _mm_movemask_epi8(__m128i a) { __vector unsigned long long result; const __vector unsigned char perm_mask = @@ -313,15 +313,15 @@ Y_FORCE_INLINE int _mm_movemask_epi8(__m128i a) { #elif __BIG_ENDIAN__ return result[0]; #endif -} - +} + Y_FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) { __v4si result; result = vec_cts((__v4sf)a, 0); return (__m128i)result; -} - +} + #define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ do { \ __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ @@ -338,27 +338,27 @@ Y_FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) { (row3) = (__v4sf)vec_mergel((vector long long)__t2, \ (vector long long)__t3); \ } while (0) - + Y_FORCE_INLINE __m128i _mm_or_si128(__m128i a, __m128i b) { return (__m128i)vec_or((__v2di)a, (__v2di)b); -} - +} + Y_FORCE_INLINE __m128i _mm_and_si128(__m128i a, __m128i b) { return (__m128i)vec_and((__v2di)a, (__v2di)b); -} - +} + Y_FORCE_INLINE __m128i _mm_andnot_si128(__m128i a, __m128i b) { return (__m128i)vec_andc((__v2di)b, (__v2di)a); -} - +} + Y_FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) { return (__m128i)vec_xor((__v2di)a, (__v2di)b); } Y_FORCE_INLINE __m128i _mm_setzero_si128() { return (__m128i)(__v4si){0, 0, 0, 0}; -} - +} + Y_FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i op1, long op2) { unsigned long element_selector_10 = op2 & 0x03; unsigned long element_selector_32 = (op2 >> 2) & 0x03; @@ -373,7 +373,7 @@ Y_FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i op1, long op2) { #endif }; __v4su t; - + #ifdef __LITTLE_ENDIAN__ t[0] = permute_selectors[element_selector_10]; t[1] = permute_selectors[element_selector_32]; @@ -387,11 +387,11 @@ Y_FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i op1, long op2) { #endif return (__m128i)vec_perm((__v4si)op1, (__v4si)op1, (__vector unsigned char)t); } - + Y_FORCE_INLINE int _mm_extract_epi16(__m128i a, int imm) { return (unsigned short)((__v8hi)a)[imm & 7]; -} - +} + Y_FORCE_INLINE int _mm_extract_epi8(__m128i a, int imm) { return (unsigned char)((__v16qi)a)[imm & 15]; } @@ -423,8 +423,8 @@ Y_FORCE_INLINE __m128i _mm_slli_epi16(__m128i a, int count) { } return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int count) { __v4su lshift; __v4si result = {0, 0, 0, 0}; @@ -440,8 +440,8 @@ Y_FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int count) { } return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int count) { __v2du lshift; __v2di result = {0, 0}; @@ -457,8 +457,8 @@ Y_FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int count) { } return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_slli_si128(__m128i a, int imm) { __v16qu result; const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; @@ -473,8 +473,8 @@ Y_FORCE_INLINE __m128i _mm_slli_si128(__m128i a, int imm) { result = zeros; return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_srli_epi16(__m128i a, int count) { if ((unsigned long)count >= 16) { /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ @@ -487,8 +487,8 @@ Y_FORCE_INLINE __m128i _mm_srli_epi16(__m128i a, int count) { replicated_count = vec_splats((short)count); return (__m128i)vec_sr((vector signed short)a, replicated_count); } -} - +} + Y_FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, int count) { if ((unsigned long)count >= 32) { /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ @@ -501,8 +501,8 @@ Y_FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, int count) { replicated_count = vec_splats(count); return (__m128i)vec_sr((vector signed int)a, replicated_count); } -} - +} + Y_FORCE_INLINE __m128i _mm_srli_epi64(__m128i a, int count) { if ((unsigned long)count >= 64) { /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ @@ -521,8 +521,8 @@ Y_FORCE_INLINE __m128i _mm_srli_epi64(__m128i a, int count) { mask[1] = m; return vec_and(vec_srl(vec_sro(a, (__m128i)replicated_count), (__m128i)replicated_count), (__v16qu)mask); } -} - +} + Y_FORCE_INLINE __m128i _mm_bsrli_si128(__m128i a, const int __N) { __v16qu result; const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; @@ -545,8 +545,8 @@ Y_FORCE_INLINE __m128i _mm_bsrli_si128(__m128i a, const int __N) { Y_FORCE_INLINE __m128i _mm_srli_si128(__m128i a, int imm) { return _mm_bsrli_si128(a, imm); -} - +} + Y_FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) { __v8hu rshift = {15, 15, 15, 15, 15, 15, 15, 15}; __v8hi result; @@ -559,10 +559,10 @@ Y_FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) { } result = vec_vsrah((__v8hi)a, rshift); return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, int count) { - // return vec_shiftrightarithmetic4wimmediate(a, count); //!< Failes to work with count >= 32. + // return vec_shiftrightarithmetic4wimmediate(a, count); //!< Failes to work with count >= 32. __v4su rshift = {31, 31, 31, 31}; __v4si result; @@ -576,16 +576,16 @@ Y_FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, int count) { } else { rshift = vec_splats((unsigned int)count); } - } + } result = vec_vsraw((__v4si)a, rshift); return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) { __v8hu lshift, shmask; const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15}; __v8hu result; - + #ifdef __LITTLE_ENDIAN__ lshift = vec_splat((__v8hu)count, 0); #elif __BIG_ENDIAN__ @@ -595,8 +595,8 @@ Y_FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) { result = vec_vslh((__v8hu)a, lshift); result = vec_sel(shmask, result, shmask); return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) { __v4su lshift, shmask; const __v4su shmax = {32, 32, 32, 32}; @@ -609,23 +609,23 @@ Y_FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) { shmask = vec_cmplt(lshift, shmax); result = vec_vslw((__v4su)a, lshift); result = vec_sel(shmask, result, shmask); - + return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) { __v2du lshift, shmask; const __v2du shmax = {64, 64}; __v2du result; - + lshift = (__v2du)vec_splat((__v2du)count, 0); shmask = vec_cmplt(lshift, shmax); result = vec_sl((__v2du)a, lshift); result = ((vector long long)shmask & ~(vector long long)shmask) | ((vector long long)result & (vector long long)shmask); return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) { __v8hu rshift, shmask; const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15}; @@ -641,8 +641,8 @@ Y_FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) { result = vec_sel(shmask, result, shmask); return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) { __v4su rshift, shmask; const __v4su shmax = {32, 32, 32, 32}; @@ -658,8 +658,8 @@ Y_FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) { result = vec_sel(shmask, result, shmask); return (__m128i)result; -} - +} + Y_FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) { __v2du rshift, shmask; const __v2du shmax = {64, 64}; @@ -671,93 +671,93 @@ Y_FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) { result = (__v2du)vec_sel((__v2du)shmask, (__v2du)result, (__v2du)shmask); return (__m128i)result; -} - +} + Y_FORCE_INLINE void _mm_storeu_si128(__m128i* p, __m128i a) { vec_vsx_st(a, 0, p); -} - +} + Y_FORCE_INLINE void _mm_store_si128(__m128i* p, __m128i a) { vec_st((__v16qu)a, 0, (__v16qu*)p); -} - +} + Y_FORCE_INLINE __m128i _mm_unpackhi_epi8(__m128i a, __m128i b) { return (__m128i)vec_mergel((__v16qu)a, (__v16qu)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpackhi_epi16(__m128i a, __m128i b) { return (__m128i)vec_mergel((__v8hu)a, (__v8hu)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpackhi_epi32(__m128i a, __m128i b) { return (__m128i)vec_mergel((__v4su)a, (__v4su)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) { return (__m128i)vec_mergel((vector long long)a, (vector long long)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpacklo_epi8(__m128i a, __m128i b) { return (__m128i)vec_mergeh((__v16qu)a, (__v16qu)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpacklo_epi16(__m128i a, __m128i b) { return (__m128i)vec_mergeh((__v8hi)a, (__v8hi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpacklo_epi32(__m128i a, __m128i b) { return (__m128i)vec_mergeh((__v4si)a, (__v4si)b); -} - +} + Y_FORCE_INLINE __m128i _mm_unpacklo_epi64(__m128i a, __m128i b) { return (__m128i)vec_mergeh((vector long long)a, (vector long long)b); -} - +} + Y_FORCE_INLINE __m128i _mm_add_epi8(__m128i a, __m128i b) { return (__m128i)((__v16qu)a + (__v16qu)b); } Y_FORCE_INLINE __m128i _mm_add_epi16(__m128i a, __m128i b) { return (__m128i)((__v8hu)a + (__v8hu)b); -} - +} + Y_FORCE_INLINE __m128i _mm_add_epi32(__m128i a, __m128i b) { return (__m128i)((__v4su)a + (__v4su)b); -} - +} + Y_FORCE_INLINE __m128i _mm_add_epi64(__m128i a, __m128i b) { return (__m128i)((__v2du)a + (__v2du)b); -} - +} + Y_FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) { const vector signed int zero = {0, 0, 0, 0}; return (__m128i)vec_vmsumshm((__v8hi)a, (__v8hi)b, zero); -} - +} + Y_FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) { return (__m128i)((__v16qu)a - (__v16qu)b); } Y_FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) { return (__m128i)((__v8hu)a - (__v8hu)b); -} - +} + Y_FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) { return (__m128i)((__v4su)a - (__v4su)b); -} - +} + Y_FORCE_INLINE __m128i _mm_sub_epi64(__m128i a, __m128i b) { return (__m128i)((__v2du)a - (__v2du)b); -} - +} + Y_FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128 b) { #ifdef __LITTLE_ENDIAN__ return (__m128i)vec_mule((__v4su)a, (__v4su)b); #elif __BIG_ENDIAN__ return (__m128i)vec_mulo((__v4su)a, (__v4su)b); #endif -} - +} + Y_FORCE_INLINE __m128i _mm_set_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) { return (__m128i)(__v16qi){q00, q01, q02, q03, q04, q05, q06, q07, q08, q09, q10, q11, q12, q13, q14, q15}; }; @@ -768,16 +768,16 @@ Y_FORCE_INLINE __m128i _mm_setr_epi8(char q15, char q14, char q13, char q12, cha Y_FORCE_INLINE __m128i _mm_set_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) { return (__m128i)(__v8hi){q0, q1, q2, q3, q4, q5, q6, q7}; -} - +} + Y_FORCE_INLINE __m128i _mm_setr_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) { return (__m128i)(__v8hi){q7, q6, q5, q4, q3, q2, q1, q0}; } Y_FORCE_INLINE __m128i _mm_set_epi32(int q3, int q2, int q1, int q0) { return (__m128i)(__v4si){q0, q1, q2, q3}; -} - +} + Y_FORCE_INLINE __m128i _mm_setr_epi32(int q3, int q2, int q1, int q0) { return (__m128i)(__v4si){q3, q2, q1, q0}; } @@ -796,52 +796,52 @@ Y_FORCE_INLINE __m128i _mm_set1_epi32(int a) { Y_FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) { return (__m128i)vec_cmpeq((__v16qi)a, (__v16qi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmpeq_epi16(__m128i a, __m128i b) { return (__m128i)vec_cmpeq((__v8hi)a, (__v8hi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i a, __m128i b) { return (__m128i)vec_cmpeq((__v4si)a, (__v4si)b); -} - +} + Y_FORCE_INLINE __m128i _mm_packs_epi16(__m128i a, __m128i b) { return (__m128i)vec_packs((__v8hi)a, (__v8hi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) { return (__m128i)vec_packs((__v4si)a, (__v4si)b); -} - +} + Y_FORCE_INLINE __m128i _mm_packus_epi16(__m128i a, __m128i b) { return (__m128i)vec_packsu((vector signed short)a, (vector signed short)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cvtsi64_si128(i64 a) { return (__m128i)(__v2di){a, 0LL}; -} - +} + Y_FORCE_INLINE __m128i _mm_cvtsi32_si128(int a) { return _mm_set_epi32(0, 0, 0, a); -} - +} + Y_FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) { return ((__v4si)a)[0]; -} - +} + Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) { return ((__v2di)a)[0]; -} - +} + Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* p) { return *p; -} - +} + Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* p) { return (__m128i)(vec_vsx_ld(0, (signed int const*)p)); -} - +} + Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* p) { return _mm_loadu_si128(p); } @@ -887,43 +887,43 @@ Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) { Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i a) { return (__m128)a; -} - +} + Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 a) { return (__m128i)a; -} - +} + Y_FORCE_INLINE __m128i _mm_cmpgt_epi8(__m128i a, __m128i b) { return (__m128i)vec_cmpgt((__v16qi)a, (__v16qi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmpgt_epi16(__m128i a, __m128i b) { return (__m128i)vec_cmpgt((__v8hi)a, (__v8hi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmpgt_epi32(__m128i a, __m128i b) { return (__m128i)vec_cmpgt((__v4si)a, (__v4si)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmpgt_epi64(__m128i a, __m128i b) { return vec_cmpgt((vector signed long long)a, (vector signed long long)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) { return (__m128i)vec_cmplt((__v16qi)a, (__v16qi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmplt_epi16(__m128i a, __m128i b) { return (__m128i)vec_cmplt((__v8hi)a, (__v8hi)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmplt_epi32(__m128i a, __m128i b) { return (__m128i)vec_cmplt((__v4si)a, (__v4si)b); -} - +} + Y_FORCE_INLINE __m128i _mm_cmplt_epi64(__m128i a, __m128i b) { return vec_cmplt((vector signed long long)a, (vector signed long long)b); -} +} Y_FORCE_INLINE __m128i _mm_sad_epu8(__m128i A, __m128i B) { __v16qu a, b; diff --git a/library/cpp/sse/sse.h b/library/cpp/sse/sse.h index d8bcbd2d4e..19bac17de0 100644 --- a/library/cpp/sse/sse.h +++ b/library/cpp/sse/sse.h @@ -14,7 +14,7 @@ #include <xmmintrin.h> #include <emmintrin.h> #include <pmmintrin.h> -#define ARCADIA_SSE +#define ARCADIA_SSE #if defined(_ssse3_) #include <tmmintrin.h> #endif @@ -26,8 +26,8 @@ #endif #elif defined(_arm64_) #include "sse2neon.h" -#define ARCADIA_SSE -#elif defined(_ppc64_) -#include "powerpc.h" -#define ARCADIA_SSE +#define ARCADIA_SSE +#elif defined(_ppc64_) +#include "powerpc.h" +#define ARCADIA_SSE #endif diff --git a/library/cpp/sse/sse2neon.h b/library/cpp/sse/sse2neon.h index 8d9437d985..695dbd3041 100644 --- a/library/cpp/sse/sse2neon.h +++ b/library/cpp/sse/sse2neon.h @@ -42,21 +42,21 @@ union __m128i { union __m128 { float32x4_t AsFloat32x4; float64x2_t AsFloat64x2; - - uint32x4_t AsUi32x4; - int32x4_t AsSi32x4; - + + uint32x4_t AsUi32x4; + int32x4_t AsSi32x4; + uint64x2_t AsUi64x2; int64x2_t AsSi64x2; - + uint8x16_t AsUi8x16; - int8x16_t AsSi8x16; - - __m128i As128i; + int8x16_t AsSi8x16; + + __m128i As128i; }; -typedef float64x2_t __m128d; - +typedef float64x2_t __m128d; + enum _mm_hint { /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ @@ -194,37 +194,37 @@ struct TWrapperSingleNegDup: public TBaseWrapper<__m128i> { } }; -inline __m128i _mm_srl_epi16(__m128i a, __m128i count) { - __m128i res; - res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(-count.AsUi16x8[0])); - return res; -} - - -inline __m128i _mm_srl_epi32(__m128i a, __m128i count) { - __m128i res; - res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(-count.AsUi32x4[0])); - return res; -} - -inline __m128i _mm_srl_epi64(__m128i a, __m128i count) { - __m128i res; - res.AsUi64x2 = vshlq_u64(a.AsUi64x2, vdupq_n_s64(-count.AsUi64x2[0])); - return res; -} - -inline __m128i _mm_srai_epi16(__m128i a, int count) { - __m128i res; - res.AsSi16x8 = vqshlq_s16(a.AsSi16x8, vdupq_n_s16(-count)); - return res; -} - -inline __m128i _mm_srai_epi32(__m128i a, int count) { - __m128i res; - res.AsSi32x4 = vqshlq_s32(a.AsSi32x4, vdupq_n_s32(-count)); - return res; -} - +inline __m128i _mm_srl_epi16(__m128i a, __m128i count) { + __m128i res; + res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(-count.AsUi16x8[0])); + return res; +} + + +inline __m128i _mm_srl_epi32(__m128i a, __m128i count) { + __m128i res; + res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(-count.AsUi32x4[0])); + return res; +} + +inline __m128i _mm_srl_epi64(__m128i a, __m128i count) { + __m128i res; + res.AsUi64x2 = vshlq_u64(a.AsUi64x2, vdupq_n_s64(-count.AsUi64x2[0])); + return res; +} + +inline __m128i _mm_srai_epi16(__m128i a, int count) { + __m128i res; + res.AsSi16x8 = vqshlq_s16(a.AsSi16x8, vdupq_n_s16(-count)); + return res; +} + +inline __m128i _mm_srai_epi32(__m128i a, int count) { + __m128i res; + res.AsSi32x4 = vqshlq_s32(a.AsSi32x4, vdupq_n_s32(-count)); + return res; +} + using _mm_srli_epi16 = TWrapperSingleNegDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16, decltype(vdupq_n_s16), vdupq_n_s16>; @@ -236,25 +236,25 @@ using _mm_srli_epi64 = decltype(vdupq_n_s64), vdupq_n_s64>; -inline __m128i _mm_sll_epi16(__m128i a, __m128i count) { - __m128i res; - res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(count.AsUi16x8[0])); - return res; -} - - -inline __m128i _mm_sll_epi32(__m128i a, __m128i count) { - __m128i res; - res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(count.AsUi32x4[0])); - return res; -} - -inline __m128i _mm_sll_epi64(__m128i a, __m128i count) { - __m128i res; - res.AsUi64x2 = vshlq_u64(a.AsUi64x2, vdupq_n_s64(count.AsUi64x2[0])); - return res; -} - +inline __m128i _mm_sll_epi16(__m128i a, __m128i count) { + __m128i res; + res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(count.AsUi16x8[0])); + return res; +} + + +inline __m128i _mm_sll_epi32(__m128i a, __m128i count) { + __m128i res; + res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(count.AsUi32x4[0])); + return res; +} + +inline __m128i _mm_sll_epi64(__m128i a, __m128i count) { + __m128i res; + res.AsUi64x2 = vshlq_u64(a.AsUi64x2, vdupq_n_s64(count.AsUi64x2[0])); + return res; +} + using _mm_slli_epi16 = TWrapperSingleDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16, decltype(vdupq_n_s16), vdupq_n_s16>; @@ -287,10 +287,10 @@ struct TWrapperDualSwap : TBaseWrapper<__m128i> { } }; -template <typename TOp, typename TFunc, TFunc* func, typename TArgument = __m128> -struct TWrapperDualF : TBaseWrapper<TArgument> { +template <typename TOp, typename TFunc, TFunc* func, typename TArgument = __m128> +struct TWrapperDualF : TBaseWrapper<TArgument> { Y_FORCE_INLINE - TWrapperDualF(const TArgument& op1, const TArgument& op2) { + TWrapperDualF(const TArgument& op1, const TArgument& op2) { TQType<TOp>::As(TBaseWrapper<TArgument>::Value) = (TOp) func(TQType<TOp>::As(op1), TQType<TOp>::As(op2)); } }; @@ -306,27 +306,27 @@ using _mm_add_epi16 = TWrapperDual<uint16x8_t, decltype(vaddq_u16), vaddq_u16>; using _mm_add_epi32 = TWrapperDual<uint32x4_t, decltype(vaddq_u32), vaddq_u32>; using _mm_add_epi64 = TWrapperDual<uint64x2_t, decltype(vaddq_u64), vaddq_u64>; -inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { - int32x4_t aLow; - int32x4_t aHigh; - int32x4_t bLow; - int32x4_t bHigh; +inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { + int32x4_t aLow; + int32x4_t aHigh; + int32x4_t bLow; + int32x4_t bHigh; #ifdef __LITTLE_ENDIAN__ aLow[0] = a.AsSi16x8[0]; //!< I couldn't find vector instructions to do that. Feel free to fix this code. aLow[1] = a.AsSi16x8[2]; aLow[2] = a.AsSi16x8[4]; aLow[3] = a.AsSi16x8[6]; - + aHigh[0] = a.AsSi16x8[1]; aHigh[1] = a.AsSi16x8[3]; aHigh[2] = a.AsSi16x8[5]; aHigh[3] = a.AsSi16x8[7]; - + bLow[0] = b.AsSi16x8[0]; bLow[1] = b.AsSi16x8[2]; bLow[2] = b.AsSi16x8[4]; bLow[3] = b.AsSi16x8[6]; - + bHigh[0] = b.AsSi16x8[1]; bHigh[1] = b.AsSi16x8[3]; bHigh[2] = b.AsSi16x8[5]; @@ -334,14 +334,14 @@ inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { #else #error Not implemented yet. Do it yourself. #endif - - const int32x4_t lowMul = vmulq_u32(aLow, bLow); - const int32x4_t highMul = vmulq_u32(aHigh, bHigh); - __m128i res; - res.AsSi32x4 = vaddq_u32(lowMul, highMul); - return res; -} - + + const int32x4_t lowMul = vmulq_u32(aLow, bLow); + const int32x4_t highMul = vmulq_u32(aHigh, bHigh); + __m128i res; + res.AsSi32x4 = vaddq_u32(lowMul, highMul); + return res; +} + using _mm_sub_epi8 = TWrapperDual<uint8x16_t, decltype(vsubq_u8), vsubq_u8>; using _mm_sub_epi16 = TWrapperDual<uint16x8_t, decltype(vsubq_u16), vsubq_u16>; using _mm_sub_epi32 = TWrapperDual<uint32x4_t, decltype(vsubq_u32), vsubq_u32>; @@ -455,28 +455,28 @@ struct _mm_storel_epi64 : TBaseWrapper<__m128i> { } }; -struct ShuffleStruct4 { - ui8 x[4]; -}; - +struct ShuffleStruct4 { + ui8 x[4]; +}; + Y_FORCE_INLINE ShuffleStruct4 -_MM_SHUFFLE(ui8 x4, ui8 x3, ui8 x2, ui8 x1) { - ShuffleStruct4 result; - result.x[0] = x1; - result.x[1] = x2; - result.x[2] = x3; - result.x[3] = x4; - return result; -} - +_MM_SHUFFLE(ui8 x4, ui8 x3, ui8 x2, ui8 x1) { + ShuffleStruct4 result; + result.x[0] = x1; + result.x[1] = x2; + result.x[2] = x3; + result.x[3] = x4; + return result; +} + Y_FORCE_INLINE __m128i _mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) { __m128i result; - const ui8 xi[4] = { - ui8(op2.x[0] * 4), ui8(op2.x[1] * 4), + const ui8 xi[4] = { + ui8(op2.x[0] * 4), ui8(op2.x[1] * 4), ui8(op2.x[2] * 4), ui8(op2.x[3] * 4) }; - const uint8x16_t transform = { + const uint8x16_t transform = { ui8(xi[0]), ui8(xi[0] + 1), ui8(xi[0] + 2), ui8(xi[0] + 3), ui8(xi[1]), ui8(xi[1] + 1), ui8(xi[1] + 2), ui8(xi[1] + 3), ui8(xi[2]), ui8(xi[2] + 1), ui8(xi[2] + 2), ui8(xi[2] + 3), @@ -504,39 +504,39 @@ template <int imm> struct THelper_mm_srli_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE THelper_mm_srli_si128(const __m128i a) { - const auto zero = vdupq_n_u8(0); + const auto zero = vdupq_n_u8(0); TQType<uint8x16_t>::As(Value) = vextq_u8(a.AsUi8x16, zero, imm); } }; -template <> -struct THelper_mm_srli_si128<16> : TBaseWrapper<__m128i> { - Y_FORCE_INLINE - THelper_mm_srli_si128(const __m128i /* a */) { - const auto zero = vdupq_n_u8(0); - TQType<uint8x16_t>::As(Value) = zero; - } -}; - +template <> +struct THelper_mm_srli_si128<16> : TBaseWrapper<__m128i> { + Y_FORCE_INLINE + THelper_mm_srli_si128(const __m128i /* a */) { + const auto zero = vdupq_n_u8(0); + TQType<uint8x16_t>::As(Value) = zero; + } +}; + #define _mm_srli_si128(a, imm) THelper_mm_srli_si128<imm>(a) template<int imm> -inline uint8x16_t vextq_u8_function(uint8x16_t a, uint8x16_t b) { - return vextq_u8(a, b, imm); -} - +inline uint8x16_t vextq_u8_function(uint8x16_t a, uint8x16_t b) { + return vextq_u8(a, b, imm); +} + template<> -inline uint8x16_t vextq_u8_function<16>(uint8x16_t /* a */, uint8x16_t b) { - return b; -} - +inline uint8x16_t vextq_u8_function<16>(uint8x16_t /* a */, uint8x16_t b) { + return b; +} + template <int imm> struct THelper_mm_slli_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE THelper_mm_slli_si128(const __m128i a) { auto zero = vdupq_n_u8(0); - TQType<uint8x16_t>::As(Value) = vextq_u8_function<16 - imm>(zero, a.AsUi8x16); + TQType<uint8x16_t>::As(Value) = vextq_u8_function<16 - imm>(zero, a.AsUi8x16); } }; @@ -598,14 +598,14 @@ struct _mm_cvtsi32_si128 : TBaseWrapper<__m128i> { } }; -struct _mm_cvtsi64_si128 : TBaseWrapper<__m128i> { - Y_FORCE_INLINE - _mm_cvtsi64_si128(i64 op) { - auto zero = vdupq_n_s64(0); - TQType<int64x2_t>::As(Value) = vsetq_lane_s64(op, zero, 0); - } -}; - +struct _mm_cvtsi64_si128 : TBaseWrapper<__m128i> { + Y_FORCE_INLINE + _mm_cvtsi64_si128(i64 op) { + auto zero = vdupq_n_s64(0); + TQType<int64x2_t>::As(Value) = vsetq_lane_s64(op, zero, 0); + } +}; + template <typename TOpOut, typename TOpIn, typename TFunc, TFunc* func, typename TCombine, TCombine* combine> @@ -648,7 +648,7 @@ int extract_epi8_arm(__m128i arg) { template<int imm> int extract_epi16_arm(__m128i arg) { - return vgetq_lane_u16(arg.AsUi16x8, imm); + return vgetq_lane_u16(arg.AsUi16x8, imm); } template<int imm> @@ -712,14 +712,14 @@ struct TQType<float64x2_t> { static inline const float64x2_t& As(const __m128i& value) { return value.AsFloat64x2; } - - static inline float64x2_t& As(__m128d& value) { - return value; - } - - static inline const float64x2_t& As(const __m128d& value) { - return value; - } + + static inline float64x2_t& As(__m128d& value) { + return value; + } + + static inline const float64x2_t& As(const __m128d& value) { + return value; + } }; using _mm_set1_ps = TWrapperSimpleF<float32x4_t, @@ -735,9 +735,9 @@ struct _mm_setzero_ps : TBaseWrapper<__m128> { }; Y_FORCE_INLINE __m128d _mm_setzero_pd() { - return vdupq_n_f64(0.); -} - + return vdupq_n_f64(0.); +} + Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* ptr) { __m128 result; result.AsFloat32x4 = vld1q_f32(ptr); @@ -745,19 +745,19 @@ Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* ptr) { } Y_FORCE_INLINE __m128 _mm_load_ps(const float* ptr) { - __m128 result; - result.AsFloat32x4 = vld1q_f32(ptr); - return result; -} - + __m128 result; + result.AsFloat32x4 = vld1q_f32(ptr); + return result; +} + Y_FORCE_INLINE void _mm_storeu_ps(float* ptr, const __m128& op) { vst1q_f32(ptr, op.AsFloat32x4); } Y_FORCE_INLINE void _mm_store_ps(float* ptr, const __m128& op) { - vst1q_f32(ptr, op.AsFloat32x4); -} - + vst1q_f32(ptr, op.AsFloat32x4); +} + struct _mm_set_ps : TBaseWrapper<__m128> { Y_FORCE_INLINE _mm_set_ps(const float x3, const float x2, @@ -769,25 +769,25 @@ struct _mm_set_ps : TBaseWrapper<__m128> { }; Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { - const float64x1_t p0 = {d0}; - const float64x1_t p1 = {d1}; - return vcombine_f64(p0, p1); -} - + const float64x1_t p0 = {d0}; + const float64x1_t p1 = {d1}; + return vcombine_f64(p0, p1); +} + Y_FORCE_INLINE __m128d _mm_loadu_pd(const double* d) { - __m128d res; - res = vld1q_f64(d); - return res; -} - + __m128d res; + res = vld1q_f64(d); + return res; +} + Y_FORCE_INLINE void _mm_storeu_pd(double* res, __m128d a) { - vst1q_f64(res, a); -} - + vst1q_f64(res, a); +} + Y_FORCE_INLINE void _mm_store_pd(double* res, __m128d a) { - vst1q_f64(res, a); -} - + vst1q_f64(res, a); +} + using _mm_add_ps = TWrapperDualF<float32x4_t, decltype(vaddq_f32), vaddq_f32>; using _mm_sub_ps = TWrapperDualF<float32x4_t, decltype(vsubq_f32), vsubq_f32>; using _mm_mul_ps = TWrapperDualF<float32x4_t, decltype(vmulq_f32), vmulq_f32>; @@ -797,11 +797,11 @@ using _mm_cmpgt_ps = TWrapperDualF<float32x4_t, decltype(vcgtq_f32), vcgtq_f32>; using _mm_max_ps = TWrapperDualF<float32x4_t, decltype(vmaxq_f32), vmaxq_f32>; using _mm_min_ps = TWrapperDualF<float32x4_t, decltype(vminq_f32), vminq_f32>; -using _mm_add_pd = TWrapperDualF<float64x2_t, decltype(vaddq_f64), vaddq_f64, __m128d>; -using _mm_sub_pd = TWrapperDualF<float64x2_t, decltype(vsubq_f64), vsubq_f64, __m128d>; -using _mm_mul_pd = TWrapperDualF<float64x2_t, decltype(vmulq_f64), vmulq_f64, __m128d>; -using _mm_div_pd = TWrapperDualF<float64x2_t, decltype(vdivq_f64), vdivq_f64, __m128d>; - +using _mm_add_pd = TWrapperDualF<float64x2_t, decltype(vaddq_f64), vaddq_f64, __m128d>; +using _mm_sub_pd = TWrapperDualF<float64x2_t, decltype(vsubq_f64), vsubq_f64, __m128d>; +using _mm_mul_pd = TWrapperDualF<float64x2_t, decltype(vmulq_f64), vmulq_f64, __m128d>; +using _mm_div_pd = TWrapperDualF<float64x2_t, decltype(vdivq_f64), vdivq_f64, __m128d>; + struct _mm_and_ps : TBaseWrapper<__m128> { Y_FORCE_INLINE _mm_and_ps(const __m128& op1, const __m128& op2) { @@ -863,17 +863,17 @@ struct TCvtF2SWrapperSingle : TBaseWrapper<__m128i> { } }; -inline __m128i _mm_cvtps_epi32(__m128 a) { - /// vcvtq_s32_f32 rounds to zero, but we need to round to the nearest. - static const float32x4_t half = vdupq_n_f32(0.5f); - static const float32x4_t negHalf = vdupq_n_f32(-0.5f); - static const float32x4_t zero = vdupq_n_f32(0.0f); - const float32x4_t corrections = vbslq_f32(vcgeq_f32(a.AsFloat32x4, zero), half, negHalf); - __m128i res; - res.AsSi32x4 = vcvtq_s32_f32(vaddq_f32(a.AsFloat32x4, corrections)); - return res; -} - +inline __m128i _mm_cvtps_epi32(__m128 a) { + /// vcvtq_s32_f32 rounds to zero, but we need to round to the nearest. + static const float32x4_t half = vdupq_n_f32(0.5f); + static const float32x4_t negHalf = vdupq_n_f32(-0.5f); + static const float32x4_t zero = vdupq_n_f32(0.0f); + const float32x4_t corrections = vbslq_f32(vcgeq_f32(a.AsFloat32x4, zero), half, negHalf); + __m128i res; + res.AsSi32x4 = vcvtq_s32_f32(vaddq_f32(a.AsFloat32x4, corrections)); + return res; +} + using _mm_cvttps_epi32 = TCvtF2SWrapperSingle<int32x4_t, float32x4_t, decltype(vcvtq_s32_f32), vcvtq_s32_f32>; @@ -894,36 +894,36 @@ Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) { static inline void _mm_pause() { __asm__ ("YIELD"); } - -static inline __m128 _mm_rsqrt_ps(__m128 a) { - __m128 res; - res.AsFloat32x4 = vrsqrteq_f32(a.AsFloat32x4); - return res; -} - -inline float _mm_cvtss_f32(__m128 a) { - return a.AsFloat32x4[0]; -} - -inline __m128 _mm_cmpunord_ps(__m128 a, __m128 b) { - __m128 res; - res.AsUi32x4 = vorrq_u32( - vmvnq_u32(vceqq_f32(a.AsFloat32x4, a.AsFloat32x4)), //!< 0xffffffff for all nans in a. + +static inline __m128 _mm_rsqrt_ps(__m128 a) { + __m128 res; + res.AsFloat32x4 = vrsqrteq_f32(a.AsFloat32x4); + return res; +} + +inline float _mm_cvtss_f32(__m128 a) { + return a.AsFloat32x4[0]; +} + +inline __m128 _mm_cmpunord_ps(__m128 a, __m128 b) { + __m128 res; + res.AsUi32x4 = vorrq_u32( + vmvnq_u32(vceqq_f32(a.AsFloat32x4, a.AsFloat32x4)), //!< 0xffffffff for all nans in a. vmvnq_u32(vceqq_f32(b.AsFloat32x4, b.AsFloat32x4)) //!< 0xffffffff all nans in b. - ); - return res; -} - -inline __m128 _mm_andnot_ps(__m128 a, __m128 b) { - __m128 res; - res.AsFloat32x4 = vandq_u32(vmvnq_u32(a.AsUi32x4), b.AsUi32x4); - return res; -} - -inline void _mm_store_ss(float* p, __m128 a) { - *p = vgetq_lane_f32(a.AsFloat32x4, 0); -} - + ); + return res; +} + +inline __m128 _mm_andnot_ps(__m128 a, __m128 b) { + __m128 res; + res.AsFloat32x4 = vandq_u32(vmvnq_u32(a.AsUi32x4), b.AsUi32x4); + return res; +} + +inline void _mm_store_ss(float* p, __m128 a) { + *p = vgetq_lane_f32(a.AsFloat32x4, 0); +} + inline float vgetg_lane_f32_switch(float32x4_t a, ui8 b) { switch (b & 0x3) { case 0: @@ -938,20 +938,20 @@ inline float vgetg_lane_f32_switch(float32x4_t a, ui8 b) { return 0; } -inline __m128 _mm_shuffle_ps(__m128 a, __m128 b, const ShuffleStruct4& shuf) { +inline __m128 _mm_shuffle_ps(__m128 a, __m128 b, const ShuffleStruct4& shuf) { __m128 ret; ret.AsFloat32x4 = vmovq_n_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[0])); ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[1]), ret.AsFloat32x4, 1); ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[2]), ret.AsFloat32x4, 2); ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[3]), ret.AsFloat32x4, 3); return ret; -} - -inline __m128 _mm_or_ps(__m128 a, __m128 b) { - __m128 res; - res.AsUi32x4 = vorrq_u32(a.AsUi32x4, b.AsUi32x4); - return res; -} +} + +inline __m128 _mm_or_ps(__m128 a, __m128 b) { + __m128 res; + res.AsUi32x4 = vorrq_u32(a.AsUi32x4, b.AsUi32x4); + return res; +} inline __m128i _mm_sad_epu8(__m128i a, __m128i b) { uint16x8_t t = vpaddlq_u8(vabdq_u8(a.AsUi8x16, b.AsUi8x16)); diff --git a/library/cpp/sse/ut/test.cpp b/library/cpp/sse/ut/test.cpp index 2b6a9de255..33c999d284 100644 --- a/library/cpp/sse/ut/test.cpp +++ b/library/cpp/sse/ut/test.cpp @@ -6,12 +6,12 @@ /* Author: Vitaliy Manushkin <agri@yandex-team.ru */ #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/typetraits.h> -#include <util/string/hex.h> + +#include <util/generic/typetraits.h> +#include <util/string/hex.h> #include <util/random/fast.h> #include <util/stream/output.h> - + #include <algorithm> #include <array> #include <limits> @@ -43,23 +43,23 @@ struct T_mm_CallWrapper { #include <xmmintrin.h> #include <emmintrin.h> #include <smmintrin.h> -#elif defined(_ppc64_) +#elif defined(_ppc64_) #include "library/cpp/sse/powerpc.h" -#else -#error "Unsupported platform" -#endif - -#if defined(_arm64_) -#define Wrap(T_mm_func) T_mm_func -#define WrapF(T_mm_func) T_mm_func -#define WrapD(T_mm_func) T_mm_func -#elif defined(_ppc64_) || defined(_i386_) || defined(_x86_64_) +#else +#error "Unsupported platform" +#endif + +#if defined(_arm64_) +#define Wrap(T_mm_func) T_mm_func +#define WrapF(T_mm_func) T_mm_func +#define WrapD(T_mm_func) T_mm_func +#elif defined(_ppc64_) || defined(_i386_) || defined(_x86_64_) #define Wrap(_mm_func) \ T_mm_CallWrapper<__m128i, decltype(_mm_func), _mm_func> #define WrapF(_mm_func) \ T_mm_CallWrapper<__m128, decltype(_mm_func), _mm_func> -#define WrapD(_mm_func) \ - T_mm_CallWrapper<__m128d, decltype(_mm_func), _mm_func> +#define WrapD(_mm_func) \ + T_mm_CallWrapper<__m128d, decltype(_mm_func), _mm_func> using int8x16_t = std::array<i8, 16>; using int16x8_t = std::array<i16, 8>; using int32x4_t = std::array<i32, 4>; @@ -69,7 +69,7 @@ using uint16x8_t = std::array<ui16, 8>; using uint32x4_t = std::array<ui32, 4>; using uint64x2_t = std::array<ui64, 2>; using float32x4_t = std::array<float, 4>; -using float64x2_t = std::array<double, 2>; +using float64x2_t = std::array<double, 2>; template <typename TVectorType> struct TQType { @@ -85,9 +85,9 @@ struct TQType { } static TVectorType As(__m128d param) { TVectorType value; - _mm_storeu_pd((double*)&value, param); - return value; - } + _mm_storeu_pd((double*)&value, param); + return value; + } }; #endif @@ -133,24 +133,24 @@ struct TFuncLoad<__m128> { }; template <> -struct TFuncLoad<__m128d> { - __m128d Value; - - template <typename TPointer> - TFuncLoad(TPointer* ptr) { - Value = _mm_loadu_pd((double*)ptr); - } - - operator __m128d&() { - return Value; - } - - operator const __m128d&() const { - return Value; - } -}; - -template <> +struct TFuncLoad<__m128d> { + __m128d Value; + + template <typename TPointer> + TFuncLoad(TPointer* ptr) { + Value = _mm_loadu_pd((double*)ptr); + } + + operator __m128d&() { + return Value; + } + + operator const __m128d&() const { + return Value; + } +}; + +template <> struct TFuncStore<__m128i> { template <typename TPointer> TFuncStore(TPointer* ptr, __m128i Value) { @@ -171,50 +171,50 @@ private: UNIT_TEST_SUITE(TSSEEmulTest); UNIT_TEST(Test_mm_load_si128); UNIT_TEST(Test_mm_loadu_si128); - UNIT_TEST(Test_mm_storeu_si128); - UNIT_TEST(Test_mm_loadu_si128_2); - UNIT_TEST(Test_mm_loadu_ps); - UNIT_TEST(Test_mm_storeu_ps); - - UNIT_TEST(Test_mm_slli_epi16); - UNIT_TEST(Test_mm_slli_epi32); - UNIT_TEST(Test_mm_slli_epi64); - UNIT_TEST(Test_mm_slli_si128); - + UNIT_TEST(Test_mm_storeu_si128); + UNIT_TEST(Test_mm_loadu_si128_2); + UNIT_TEST(Test_mm_loadu_ps); + UNIT_TEST(Test_mm_storeu_ps); + + UNIT_TEST(Test_mm_slli_epi16); + UNIT_TEST(Test_mm_slli_epi32); + UNIT_TEST(Test_mm_slli_epi64); + UNIT_TEST(Test_mm_slli_si128); + UNIT_TEST(Test_mm_srli_epi16); UNIT_TEST(Test_mm_srli_epi32); UNIT_TEST(Test_mm_srli_epi64); - UNIT_TEST(Test_mm_srli_si128); - - UNIT_TEST(Test_mm_srai_epi16); - UNIT_TEST(Test_mm_srai_epi32); - - UNIT_TEST(Test_mm_sll_epi16); - UNIT_TEST(Test_mm_sll_epi32); - UNIT_TEST(Test_mm_sll_epi64); - - UNIT_TEST(Test_mm_srl_epi16); - UNIT_TEST(Test_mm_srl_epi32); - UNIT_TEST(Test_mm_srl_epi64); - + UNIT_TEST(Test_mm_srli_si128); + + UNIT_TEST(Test_mm_srai_epi16); + UNIT_TEST(Test_mm_srai_epi32); + + UNIT_TEST(Test_mm_sll_epi16); + UNIT_TEST(Test_mm_sll_epi32); + UNIT_TEST(Test_mm_sll_epi64); + + UNIT_TEST(Test_mm_srl_epi16); + UNIT_TEST(Test_mm_srl_epi32); + UNIT_TEST(Test_mm_srl_epi64); + UNIT_TEST(Test_mm_add_epi16); UNIT_TEST(Test_mm_add_epi32); UNIT_TEST(Test_mm_add_epi64); UNIT_TEST(Test_mm_add_ps); - UNIT_TEST(Test_mm_add_pd); + UNIT_TEST(Test_mm_add_pd); + + UNIT_TEST(Test_mm_madd_epi16); - UNIT_TEST(Test_mm_madd_epi16); - UNIT_TEST(Test_mm_sub_epi16); UNIT_TEST(Test_mm_sub_epi32); UNIT_TEST(Test_mm_sub_epi64); UNIT_TEST(Test_mm_sub_ps); - UNIT_TEST(Test_mm_sub_pd); + UNIT_TEST(Test_mm_sub_pd); UNIT_TEST(Test_mm_mul_ps); - UNIT_TEST(Test_mm_mul_pd); + UNIT_TEST(Test_mm_mul_pd); UNIT_TEST(Test_mm_div_ps); - UNIT_TEST(Test_mm_div_pd); + UNIT_TEST(Test_mm_div_pd); UNIT_TEST(Test_mm_max_ps); UNIT_TEST(Test_mm_min_ps); UNIT_TEST(Test_mm_and_ps); @@ -250,11 +250,11 @@ private: UNIT_TEST(Test_mm_set1_epi16); UNIT_TEST(Test_mm_set1_epi32); UNIT_TEST(Test_mm_set1_ps); - UNIT_TEST(Test_mm_set_ps1); + UNIT_TEST(Test_mm_set_ps1); UNIT_TEST(Test_mm_setzero_si128); UNIT_TEST(Test_mm_setzero_ps); - UNIT_TEST(Test_mm_setzero_pd); + UNIT_TEST(Test_mm_setzero_pd); UNIT_TEST(Test_mm_storel_epi64); UNIT_TEST(Test_mm_loadl_epi64); @@ -266,15 +266,15 @@ private: UNIT_TEST(Test_mm_shuffle_epi32); UNIT_TEST(Test_mm_movemask_epi8); UNIT_TEST(Test_mm_cvtsi128_si32); - UNIT_TEST(Test_mm_cvtsi128_si64); + UNIT_TEST(Test_mm_cvtsi128_si64); UNIT_TEST(Test_mm_set_epi16); UNIT_TEST(Test_mm_set_epi32); UNIT_TEST(Test_mm_set_ps); - UNIT_TEST(Test_mm_set_pd); + UNIT_TEST(Test_mm_set_pd); UNIT_TEST(Test_mm_cvtsi32_si128); - UNIT_TEST(Test_mm_cvtsi64_si128); + UNIT_TEST(Test_mm_cvtsi64_si128); UNIT_TEST(Test_mm_packs_epi16); UNIT_TEST(Test_mm_packs_epi32); @@ -287,7 +287,7 @@ private: UNIT_TEST(Test_MM_TRANSPOSE4_PS); UNIT_TEST(Test_mm_movemask_ps); - UNIT_TEST(Test_mm_movemask_ps_2); + UNIT_TEST(Test_mm_movemask_ps_2); UNIT_TEST(Test_mm_cvtepi32_ps); UNIT_TEST(Test_mm_cvtps_epi32); @@ -298,84 +298,84 @@ private: UNIT_TEST(Test_mm_mul_epu32); - UNIT_TEST(Test_mm_cmpunord_ps); - UNIT_TEST(Test_mm_andnot_ps); - UNIT_TEST(Test_mm_shuffle_ps); + UNIT_TEST(Test_mm_cmpunord_ps); + UNIT_TEST(Test_mm_andnot_ps); + UNIT_TEST(Test_mm_shuffle_ps); UNIT_TEST(Test_mm_shuffle_pd); - UNIT_TEST(Test_mm_or_ps); - UNIT_TEST(Test_mm_store_ss); - UNIT_TEST(Test_mm_store_ps); - UNIT_TEST(Test_mm_storeu_pd); + UNIT_TEST(Test_mm_or_ps); + UNIT_TEST(Test_mm_store_ss); + UNIT_TEST(Test_mm_store_ps); + UNIT_TEST(Test_mm_storeu_pd); UNIT_TEST(Test_mm_loadu_pd); UNIT_TEST(Test_mm_rsqrt_ps); UNIT_TEST(Test_matrixnet_powerpc); - + UNIT_TEST_SUITE_END(); public: void Test_mm_load_si128(); void Test_mm_loadu_si128(); - void Test_mm_storeu_si128(); - void Test_mm_loadu_si128_2(); - void Test_mm_loadu_ps(); - void Test_mm_storeu_ps(); + void Test_mm_storeu_si128(); + void Test_mm_loadu_si128_2(); + void Test_mm_loadu_ps(); + void Test_mm_storeu_ps(); template <typename TElem, int bits, int elemCount, - typename TFunc, typename TShifter, typename TOp, typename TElemFunc> + typename TFunc, typename TShifter, typename TOp, typename TElemFunc> void Test_mm_shifter_epiXX(); - enum class EDirection { - Left, - Right - }; + enum class EDirection { + Left, + Right + }; + + struct TShiftRes { + __m128i Value[17]; + }; - struct TShiftRes { - __m128i Value[17]; - }; - void Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo); - + void Test_mm_slli_epi16(); void Test_mm_slli_epi32(); void Test_mm_slli_epi64(); - void Test_mm_slli_si128(); - - void Test_mm_srli_epi16(); - void Test_mm_srli_epi32(); - void Test_mm_srli_epi64(); - void Test_mm_srli_si128(); - - void Test_mm_srai_epi16(); - void Test_mm_srai_epi32(); - - void Test_mm_sll_epi16(); - void Test_mm_sll_epi32(); - void Test_mm_sll_epi64(); - - void Test_mm_srl_epi16(); - void Test_mm_srl_epi32(); - void Test_mm_srl_epi64(); - + void Test_mm_slli_si128(); + + void Test_mm_srli_epi16(); + void Test_mm_srli_epi32(); + void Test_mm_srli_epi64(); + void Test_mm_srli_si128(); + + void Test_mm_srai_epi16(); + void Test_mm_srai_epi32(); + + void Test_mm_sll_epi16(); + void Test_mm_sll_epi32(); + void Test_mm_sll_epi64(); + + void Test_mm_srl_epi16(); + void Test_mm_srl_epi32(); + void Test_mm_srl_epi64(); + void Test_mm_add_epi8(); void Test_mm_add_epi16(); void Test_mm_add_epi32(); void Test_mm_add_epi64(); void Test_mm_add_ps(); - void Test_mm_add_pd(); + void Test_mm_add_pd(); + + void Test_mm_madd_epi16(); - void Test_mm_madd_epi16(); - void Test_mm_sub_epi8(); void Test_mm_sub_epi16(); void Test_mm_sub_epi32(); void Test_mm_sub_epi64(); void Test_mm_sub_ps(); - void Test_mm_sub_pd(); + void Test_mm_sub_pd(); void Test_mm_mul_ps(); - void Test_mm_mul_pd(); + void Test_mm_mul_pd(); void Test_mm_div_ps(); - void Test_mm_div_pd(); + void Test_mm_div_pd(); void Test_mm_max_ps(); void Test_mm_min_ps(); void Test_mm_and_ps(); @@ -427,11 +427,11 @@ public: void Test_mm_set1_epi16(); void Test_mm_set1_epi32(); void Test_mm_set1_ps(); - void Test_mm_set_ps1(); + void Test_mm_set_ps1(); void Test_mm_setzero_si128(); void Test_mm_setzero_ps(); - void Test_mm_setzero_pd(); + void Test_mm_setzero_pd(); void Test_mm_loadl_epi64(); void Test_mm_storel_epi64(); @@ -443,15 +443,15 @@ public: void Test_mm_shuffle_epi32(); void Test_mm_movemask_epi8(); void Test_mm_cvtsi128_si32(); - void Test_mm_cvtsi128_si64(); + void Test_mm_cvtsi128_si64(); void Test_mm_set_epi16(); void Test_mm_set_epi32(); void Test_mm_set_ps(); - void Test_mm_set_pd(); + void Test_mm_set_pd(); void Test_mm_cvtsi32_si128(); - void Test_mm_cvtsi64_si128(); + void Test_mm_cvtsi64_si128(); template <typename TElem, typename TNarrow, unsigned elemCount, typename TFunc> @@ -467,7 +467,7 @@ public: void Test_MM_TRANSPOSE4_PS(); void Test_mm_movemask_ps(); - void Test_mm_movemask_ps_2(); + void Test_mm_movemask_ps_2(); template <typename TFrom, typename TTo, unsigned elemCount, typename TLoadVector, typename TResultVector, @@ -484,15 +484,15 @@ public: void Test_mm_castps_si128(); void Test_mm_mul_epu32(); - - void Test_mm_cmpunord_ps(); - void Test_mm_store_ss(); - void Test_mm_store_ps(); - void Test_mm_storeu_pd(); - void Test_mm_andnot_ps(); - void Test_mm_shuffle_ps(); + + void Test_mm_cmpunord_ps(); + void Test_mm_store_ss(); + void Test_mm_store_ps(); + void Test_mm_storeu_pd(); + void Test_mm_andnot_ps(); + void Test_mm_shuffle_ps(); void Test_mm_shuffle_pd(); - void Test_mm_or_ps(); + void Test_mm_or_ps(); void Test_mm_loadu_pd(); void Test_mm_rsqrt_ps(); void Test_mm_rsqrt_ss(); @@ -502,7 +502,7 @@ public: UNIT_TEST_SUITE_REGISTRATION(TSSEEmulTest); void TSSEEmulTest::Test_mm_load_si128() { - alignas(16) char data[16] = { + alignas(16) char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; __m128i value = _mm_load_si128((__m128i*)&data); @@ -511,7 +511,7 @@ void TSSEEmulTest::Test_mm_load_si128() { } void TSSEEmulTest::Test_mm_loadu_si128() { - alignas(16) char data[17] = { + alignas(16) char data[17] = { '\x66', '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; @@ -521,127 +521,127 @@ void TSSEEmulTest::Test_mm_loadu_si128() { UNIT_ASSERT(TQType<uint64x2_t>::As(value)[1] == 0x1C66775588449933UL); } -void TSSEEmulTest::Test_mm_storeu_si128() { - alignas(16) unsigned char stub[32] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 - }; - - alignas(16) unsigned char value[16] = { - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf - }; - - const __m128i val = _mm_loadu_si128((__m128i*)&value[0]); - - for (size_t shift = 0; shift != 17; ++shift) { - alignas(16) unsigned char res[sizeof(stub)]; - memcpy(res, stub, sizeof(res)); - - _mm_storeu_si128((__m128i*)&res[shift], val); - - - alignas(16) unsigned char etalon[sizeof(stub)]; - memcpy(etalon, stub, sizeof(etalon)); - for (size_t i = 0; i != sizeof(value); ++i) { - etalon[shift + i] = value[i]; - } - - for (size_t i = 0; i != sizeof(etalon) / sizeof(etalon[0]); ++i) { - UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, 32) << " vs etalon: " << HexEncode(etalon, 32)); - } - } - -} - - -void TSSEEmulTest::Test_mm_loadu_si128_2() { - alignas(16) unsigned char stub[32] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f - }; - - for (size_t shift = 0; shift != 17; ++shift) { - const __m128i val = _mm_loadu_si128((const __m128i*)&stub[shift]); - alignas(16) unsigned char res[16]; - _mm_store_si128((__m128i*)res, val); - - for (size_t i = 0; i != 16; ++i) { - UNIT_ASSERT_EQUAL_C(res[i], stub[i + shift], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16)); - } - } -} - - -void TSSEEmulTest::Test_mm_loadu_ps() { - alignas(16) float stub[8] = { - 0.f, 1.f, 2.f, 3.f, - 4.f, 5.f, 6.f, 7.f - }; - - for (size_t shift = 0; shift != 5; ++shift) { - const __m128 val = _mm_loadu_ps(&stub[shift]); - alignas(16) float res[4]; - _mm_store_ps(res, val); - - for (size_t i = 0; i != 4; ++i) { - UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16)); - } - } -} - - -void TSSEEmulTest::Test_mm_storeu_ps() { - alignas(16) float stub[8] = { - 0.f, 1.f, 2.f, 3.f, - 4.f, 5.f, 6.f, 7.f - }; - - alignas(16) float value[4] = { - 100.f, 101.f, 102.f, 103.f - }; - const __m128 val = _mm_load_ps(value); - - for (size_t shift = 0; shift != 5; ++shift) { - alignas(16) float res[sizeof(stub) / sizeof(stub[0])]; - memcpy(res, stub, sizeof(stub)); - - _mm_storeu_ps(&res[shift], val); - - float etalon[sizeof(stub) / sizeof(stub[0])]; - memcpy(etalon, stub, sizeof(stub)); - for (size_t i = 0; i != 4; ++i) { - etalon[i + shift] = value[i]; - } - - for (size_t i = 0; i != sizeof(stub) / sizeof(stub[0]); ++i) { - UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, sizeof(res)) << " vs etalon: " << HexEncode(etalon, sizeof(etalon))); - } - } -} - +void TSSEEmulTest::Test_mm_storeu_si128() { + alignas(16) unsigned char stub[32] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + }; + + alignas(16) unsigned char value[16] = { + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf + }; + + const __m128i val = _mm_loadu_si128((__m128i*)&value[0]); + + for (size_t shift = 0; shift != 17; ++shift) { + alignas(16) unsigned char res[sizeof(stub)]; + memcpy(res, stub, sizeof(res)); + + _mm_storeu_si128((__m128i*)&res[shift], val); + + + alignas(16) unsigned char etalon[sizeof(stub)]; + memcpy(etalon, stub, sizeof(etalon)); + for (size_t i = 0; i != sizeof(value); ++i) { + etalon[shift + i] = value[i]; + } + + for (size_t i = 0; i != sizeof(etalon) / sizeof(etalon[0]); ++i) { + UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, 32) << " vs etalon: " << HexEncode(etalon, 32)); + } + } + +} + + +void TSSEEmulTest::Test_mm_loadu_si128_2() { + alignas(16) unsigned char stub[32] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f + }; + + for (size_t shift = 0; shift != 17; ++shift) { + const __m128i val = _mm_loadu_si128((const __m128i*)&stub[shift]); + alignas(16) unsigned char res[16]; + _mm_store_si128((__m128i*)res, val); + + for (size_t i = 0; i != 16; ++i) { + UNIT_ASSERT_EQUAL_C(res[i], stub[i + shift], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16)); + } + } +} + + +void TSSEEmulTest::Test_mm_loadu_ps() { + alignas(16) float stub[8] = { + 0.f, 1.f, 2.f, 3.f, + 4.f, 5.f, 6.f, 7.f + }; + + for (size_t shift = 0; shift != 5; ++shift) { + const __m128 val = _mm_loadu_ps(&stub[shift]); + alignas(16) float res[4]; + _mm_store_ps(res, val); + + for (size_t i = 0; i != 4; ++i) { + UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16)); + } + } +} + + +void TSSEEmulTest::Test_mm_storeu_ps() { + alignas(16) float stub[8] = { + 0.f, 1.f, 2.f, 3.f, + 4.f, 5.f, 6.f, 7.f + }; + + alignas(16) float value[4] = { + 100.f, 101.f, 102.f, 103.f + }; + const __m128 val = _mm_load_ps(value); + + for (size_t shift = 0; shift != 5; ++shift) { + alignas(16) float res[sizeof(stub) / sizeof(stub[0])]; + memcpy(res, stub, sizeof(stub)); + + _mm_storeu_ps(&res[shift], val); + + float etalon[sizeof(stub) / sizeof(stub[0])]; + memcpy(etalon, stub, sizeof(stub)); + for (size_t i = 0; i != 4; ++i) { + etalon[i + shift] = value[i]; + } + + for (size_t i = 0; i != sizeof(stub) / sizeof(stub[0]); ++i) { + UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, sizeof(res)) << " vs etalon: " << HexEncode(etalon, sizeof(etalon))); + } + } +} + template<typename C> -C MakeNumber(unsigned number); - +C MakeNumber(unsigned number); + template<> -__m128i MakeNumber<__m128i>(unsigned number) { - char data[16] = {0}; - memcpy(data, &number, sizeof(number)); - - return _mm_loadu_si128((__m128i*)data); -} - +__m128i MakeNumber<__m128i>(unsigned number) { + char data[16] = {0}; + memcpy(data, &number, sizeof(number)); + + return _mm_loadu_si128((__m128i*)data); +} + template<> -unsigned MakeNumber<unsigned>(unsigned number) { - return number; -} - +unsigned MakeNumber<unsigned>(unsigned number) { + return number; +} + template <typename TElem, int bits, int elemCount, - typename TFunc, typename TShifter, typename TOp, typename TElemFunc> + typename TFunc, typename TShifter, typename TOp, typename TElemFunc> void TSSEEmulTest::Test_mm_shifter_epiXX() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -650,205 +650,205 @@ void TSSEEmulTest::Test_mm_shifter_epiXX() { __m128i value = _mm_loadu_si128((__m128i*)&data); - for (unsigned shifter = 0; shifter <= bits; ++shifter) { + for (unsigned shifter = 0; shifter <= bits; ++shifter) { TElem shiftedData[elemCount]; - for (unsigned i = 0; i < elemCount; ++i) { + for (unsigned i = 0; i < elemCount; ++i) { shiftedData[i] = TElemFunc::Call(dataw[i], shifter); - } + } - const TShifter adhoc_shifter = MakeNumber<TShifter>(shifter); - - __m128i result = TFunc(value, adhoc_shifter); - - for (unsigned i = 0; i < elemCount; ++i) { + const TShifter adhoc_shifter = MakeNumber<TShifter>(shifter); + + __m128i result = TFunc(value, adhoc_shifter); + + for (unsigned i = 0; i < elemCount; ++i) { UNIT_ASSERT_EQUAL(shiftedData[i], TQType<TOp>::As(result)[i]); - } + } } } void TSSEEmulTest::Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo) { - const char data[48] = { - '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', - '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C', - '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + const char data[48] = { + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00' }; - - const __m128i a = _mm_loadu_si128((__m128i*)(data + 16)); - const TShiftRes res = foo(a); - for (int shift = 0; shift <= 16; ++shift) { - const int etalon_offset = 16 + (direction == EDirection::Left ? -shift : shift); //!< specific to little endian byte order. - const char* etalon = data + etalon_offset; - const char* res_bytes = (const char*)&res.Value[shift]; - - for (size_t byte = 0; byte != 16; ++byte) { - UNIT_ASSERT_EQUAL(etalon[byte], res_bytes[byte]); - } - } -} - -template <typename TElem> -struct THelperASHR { - static TElem Call(const TElem op, const int shift) { - constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; - if (op < 0) { - // Arithmetic shift propagates sign bit to the right - // while operator>> is implementation defined for negative values, - // so we can't use it as a reference implementation - // and we need to write some standard consistent code. - typedef TFixedWidthUnsignedInt<TElem> TUnsignedElem; - TUnsignedElem uOp(op); - const TUnsignedElem signBit = TUnsignedElem(1) << (nBitsInOp - 1); - Y_ENSURE(shift >= 0); - for (int i = 0; i != shift; ++i) { - uOp = signBit | (uOp >> 1); - } - // unsigned -> signed conversion is also implementation defined, so we need to use some other method. - return reinterpret_cast<TElem&>(uOp); - } - return shift < nBitsInOp ? op >> shift : 0; - } -}; - + + const __m128i a = _mm_loadu_si128((__m128i*)(data + 16)); + const TShiftRes res = foo(a); + for (int shift = 0; shift <= 16; ++shift) { + const int etalon_offset = 16 + (direction == EDirection::Left ? -shift : shift); //!< specific to little endian byte order. + const char* etalon = data + etalon_offset; + const char* res_bytes = (const char*)&res.Value[shift]; + + for (size_t byte = 0; byte != 16; ++byte) { + UNIT_ASSERT_EQUAL(etalon[byte], res_bytes[byte]); + } + } +} + +template <typename TElem> +struct THelperASHR { + static TElem Call(const TElem op, const int shift) { + constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; + if (op < 0) { + // Arithmetic shift propagates sign bit to the right + // while operator>> is implementation defined for negative values, + // so we can't use it as a reference implementation + // and we need to write some standard consistent code. + typedef TFixedWidthUnsignedInt<TElem> TUnsignedElem; + TUnsignedElem uOp(op); + const TUnsignedElem signBit = TUnsignedElem(1) << (nBitsInOp - 1); + Y_ENSURE(shift >= 0); + for (int i = 0; i != shift; ++i) { + uOp = signBit | (uOp >> 1); + } + // unsigned -> signed conversion is also implementation defined, so we need to use some other method. + return reinterpret_cast<TElem&>(uOp); + } + return shift < nBitsInOp ? op >> shift : 0; + } +}; + template <typename TElem> struct THelperSHR { static TElem Call(const TElem op, const int shift) { - constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; - return shift < nBitsInOp ? op >> shift : 0; + constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; + return shift < nBitsInOp ? op >> shift : 0; } }; void TSSEEmulTest::Test_mm_srli_epi16() { - Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_srli_epi16), unsigned, uint16x8_t, + Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_srli_epi16), unsigned, uint16x8_t, THelperSHR<ui16>>(); } void TSSEEmulTest::Test_mm_srli_epi32() { - Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_srli_epi32), unsigned, uint32x4_t, + Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_srli_epi32), unsigned, uint32x4_t, THelperSHR<ui32>>(); } void TSSEEmulTest::Test_mm_srli_epi64() { - Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_srli_epi64), unsigned, uint64x2_t, + Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_srli_epi64), unsigned, uint64x2_t, THelperSHR<ui64>>(); } template <typename TElem> struct THelperSHL { static TElem Call(const TElem op, const int shift) { - constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; - return shift < nBitsInOp ? op << shift : 0; + constexpr int nBitsInOp = sizeof(op) * CHAR_BIT; + return shift < nBitsInOp ? op << shift : 0; } }; void TSSEEmulTest::Test_mm_slli_epi16() { - Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_slli_epi16), unsigned, uint16x8_t, + Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_slli_epi16), unsigned, uint16x8_t, THelperSHL<ui16>>(); } void TSSEEmulTest::Test_mm_slli_epi32() { - Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_slli_epi32), unsigned, uint32x4_t, + Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_slli_epi32), unsigned, uint32x4_t, THelperSHL<ui32>>(); } void TSSEEmulTest::Test_mm_slli_epi64() { - Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_slli_epi64), unsigned, uint64x2_t, + Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_slli_epi64), unsigned, uint64x2_t, THelperSHL<ui64>>(); } -void TSSEEmulTest::Test_mm_slli_si128() { +void TSSEEmulTest::Test_mm_slli_si128() { Test_mm_byte_shifter(EDirection::Left, [] (__m128i a) -> TShiftRes { - TShiftRes res; - res.Value[0] = _mm_slli_si128(a, 0); - res.Value[1] = _mm_slli_si128(a, 1); - res.Value[2] = _mm_slli_si128(a, 2); - res.Value[3] = _mm_slli_si128(a, 3); - res.Value[4] = _mm_slli_si128(a, 4); - res.Value[5] = _mm_slli_si128(a, 5); - res.Value[6] = _mm_slli_si128(a, 6); - res.Value[7] = _mm_slli_si128(a, 7); - res.Value[8] = _mm_slli_si128(a, 8); - res.Value[9] = _mm_slli_si128(a, 9); - res.Value[10] = _mm_slli_si128(a, 10); - res.Value[11] = _mm_slli_si128(a, 11); - res.Value[12] = _mm_slli_si128(a, 12); - res.Value[13] = _mm_slli_si128(a, 13); - res.Value[14] = _mm_slli_si128(a, 14); - res.Value[15] = _mm_slli_si128(a, 15); - res.Value[16] = _mm_slli_si128(a, 16); - - return res; - }); -} - -void TSSEEmulTest::Test_mm_srl_epi16() { - Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi16), _mm_srl_epi16>, __m128i, uint16x8_t, - THelperSHR<ui16>>(); -} - -void TSSEEmulTest::Test_mm_srl_epi32() { - Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi32), _mm_srl_epi32>, __m128i, uint32x4_t, - THelperSHR<ui32>>(); -} - -void TSSEEmulTest::Test_mm_srl_epi64() { - Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi64), _mm_srl_epi64>, __m128i, uint64x2_t, - THelperSHR<ui64>>(); -} - -void TSSEEmulTest::Test_mm_srai_epi16() { - Test_mm_shifter_epiXX<i16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi16), _mm_srai_epi16>, unsigned, int16x8_t, - THelperASHR<i16>>(); -} - -void TSSEEmulTest::Test_mm_srai_epi32() { - Test_mm_shifter_epiXX<i32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi32), _mm_srai_epi32>, unsigned, int32x4_t, - THelperASHR<i32>>(); -} - -void TSSEEmulTest::Test_mm_srli_si128() { - Test_mm_byte_shifter(EDirection::Right, [](__m128i a) -> TShiftRes { - TShiftRes res; - res.Value[0] = _mm_srli_si128(a, 0); - res.Value[1] = _mm_srli_si128(a, 1); - res.Value[2] = _mm_srli_si128(a, 2); - res.Value[3] = _mm_srli_si128(a, 3); - res.Value[4] = _mm_srli_si128(a, 4); - res.Value[5] = _mm_srli_si128(a, 5); - res.Value[6] = _mm_srli_si128(a, 6); - res.Value[7] = _mm_srli_si128(a, 7); - res.Value[8] = _mm_srli_si128(a, 8); - res.Value[9] = _mm_srli_si128(a, 9); - res.Value[10] = _mm_srli_si128(a, 10); - res.Value[11] = _mm_srli_si128(a, 11); - res.Value[12] = _mm_srli_si128(a, 12); - res.Value[13] = _mm_srli_si128(a, 13); - res.Value[14] = _mm_srli_si128(a, 14); - res.Value[15] = _mm_srli_si128(a, 15); - res.Value[16] = _mm_srli_si128(a, 16); - - return res; - }); -} - -void TSSEEmulTest::Test_mm_sll_epi16() { - Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi16), _mm_sll_epi16>, __m128i, uint16x8_t, - THelperSHL<ui16>>(); -} - -void TSSEEmulTest::Test_mm_sll_epi32() { - Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi32), _mm_sll_epi32>, __m128i, uint32x4_t, - THelperSHL<ui32>>(); -} - -void TSSEEmulTest::Test_mm_sll_epi64() { - Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi64), _mm_sll_epi64>, __m128i, uint64x2_t, - THelperSHL<ui64>>(); -} - + TShiftRes res; + res.Value[0] = _mm_slli_si128(a, 0); + res.Value[1] = _mm_slli_si128(a, 1); + res.Value[2] = _mm_slli_si128(a, 2); + res.Value[3] = _mm_slli_si128(a, 3); + res.Value[4] = _mm_slli_si128(a, 4); + res.Value[5] = _mm_slli_si128(a, 5); + res.Value[6] = _mm_slli_si128(a, 6); + res.Value[7] = _mm_slli_si128(a, 7); + res.Value[8] = _mm_slli_si128(a, 8); + res.Value[9] = _mm_slli_si128(a, 9); + res.Value[10] = _mm_slli_si128(a, 10); + res.Value[11] = _mm_slli_si128(a, 11); + res.Value[12] = _mm_slli_si128(a, 12); + res.Value[13] = _mm_slli_si128(a, 13); + res.Value[14] = _mm_slli_si128(a, 14); + res.Value[15] = _mm_slli_si128(a, 15); + res.Value[16] = _mm_slli_si128(a, 16); + + return res; + }); +} + +void TSSEEmulTest::Test_mm_srl_epi16() { + Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi16), _mm_srl_epi16>, __m128i, uint16x8_t, + THelperSHR<ui16>>(); +} + +void TSSEEmulTest::Test_mm_srl_epi32() { + Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi32), _mm_srl_epi32>, __m128i, uint32x4_t, + THelperSHR<ui32>>(); +} + +void TSSEEmulTest::Test_mm_srl_epi64() { + Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi64), _mm_srl_epi64>, __m128i, uint64x2_t, + THelperSHR<ui64>>(); +} + +void TSSEEmulTest::Test_mm_srai_epi16() { + Test_mm_shifter_epiXX<i16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi16), _mm_srai_epi16>, unsigned, int16x8_t, + THelperASHR<i16>>(); +} + +void TSSEEmulTest::Test_mm_srai_epi32() { + Test_mm_shifter_epiXX<i32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi32), _mm_srai_epi32>, unsigned, int32x4_t, + THelperASHR<i32>>(); +} + +void TSSEEmulTest::Test_mm_srli_si128() { + Test_mm_byte_shifter(EDirection::Right, [](__m128i a) -> TShiftRes { + TShiftRes res; + res.Value[0] = _mm_srli_si128(a, 0); + res.Value[1] = _mm_srli_si128(a, 1); + res.Value[2] = _mm_srli_si128(a, 2); + res.Value[3] = _mm_srli_si128(a, 3); + res.Value[4] = _mm_srli_si128(a, 4); + res.Value[5] = _mm_srli_si128(a, 5); + res.Value[6] = _mm_srli_si128(a, 6); + res.Value[7] = _mm_srli_si128(a, 7); + res.Value[8] = _mm_srli_si128(a, 8); + res.Value[9] = _mm_srli_si128(a, 9); + res.Value[10] = _mm_srli_si128(a, 10); + res.Value[11] = _mm_srli_si128(a, 11); + res.Value[12] = _mm_srli_si128(a, 12); + res.Value[13] = _mm_srli_si128(a, 13); + res.Value[14] = _mm_srli_si128(a, 14); + res.Value[15] = _mm_srli_si128(a, 15); + res.Value[16] = _mm_srli_si128(a, 16); + + return res; + }); +} + +void TSSEEmulTest::Test_mm_sll_epi16() { + Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi16), _mm_sll_epi16>, __m128i, uint16x8_t, + THelperSHL<ui16>>(); +} + +void TSSEEmulTest::Test_mm_sll_epi32() { + Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi32), _mm_sll_epi32>, __m128i, uint32x4_t, + THelperSHL<ui32>>(); +} + +void TSSEEmulTest::Test_mm_sll_epi64() { + Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi64), _mm_sll_epi64>, __m128i, uint64x2_t, + THelperSHL<ui64>>(); +} + template <typename TElem> struct THelperAdd { static TElem Call(const TElem op1, const TElem op2) { @@ -873,36 +873,36 @@ void TSSEEmulTest::Test_mm_add_ps() { THelperAdd<float>, float32x4_t, __m128>(); } -void TSSEEmulTest::Test_mm_add_pd() { - Test_mm_dualop<double, 2, WrapD(_mm_add_pd), - THelperAdd<double>, float64x2_t, __m128d>(); -} - -void TSSEEmulTest::Test_mm_madd_epi16() { - alignas(16) const char data1[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', +void TSSEEmulTest::Test_mm_add_pd() { + Test_mm_dualop<double, 2, WrapD(_mm_add_pd), + THelperAdd<double>, float64x2_t, __m128d>(); +} + +void TSSEEmulTest::Test_mm_madd_epi16() { + alignas(16) const char data1[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C' }; - alignas(16) const char data2[16] = { - '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', + alignas(16) const char data2[16] = { + '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF' }; - - const __m128i value1 = TFuncLoad<__m128i>(&data1); - const __m128i value2 = TFuncLoad<__m128i>(&data2); - const __m128i res = _mm_madd_epi16(value1, value2); - - const i16* dataw1 = reinterpret_cast<const i16*>(&data1); - const i16* dataw2 = reinterpret_cast<const i16*>(&data2); - - for (size_t i = 0; i != 4; ++i) { - const size_t dataIdx = i * 2; + + const __m128i value1 = TFuncLoad<__m128i>(&data1); + const __m128i value2 = TFuncLoad<__m128i>(&data2); + const __m128i res = _mm_madd_epi16(value1, value2); + + const i16* dataw1 = reinterpret_cast<const i16*>(&data1); + const i16* dataw2 = reinterpret_cast<const i16*>(&data2); + + for (size_t i = 0; i != 4; ++i) { + const size_t dataIdx = i * 2; const i32 etalonResult = (i32) dataw1[dataIdx] * (i32) dataw2[dataIdx] + (i32) dataw1[dataIdx + 1] * (i32) dataw2[dataIdx + 1]; - const i32 value = TQType<int32x4_t>::As(res)[i]; - UNIT_ASSERT_EQUAL(value, etalonResult); - } -} - + const i32 value = TQType<int32x4_t>::As(res)[i]; + UNIT_ASSERT_EQUAL(value, etalonResult); + } +} + template <typename TElem> struct THelperSub { @@ -928,11 +928,11 @@ void TSSEEmulTest::Test_mm_sub_ps() { float32x4_t, __m128>(); } -void TSSEEmulTest::Test_mm_sub_pd() { - Test_mm_dualop<double, 2, WrapD(_mm_sub_pd), THelperSub<double>, - float64x2_t, __m128d>(); -} - +void TSSEEmulTest::Test_mm_sub_pd() { + Test_mm_dualop<double, 2, WrapD(_mm_sub_pd), THelperSub<double>, + float64x2_t, __m128d>(); +} + void TSSEEmulTest::Test_mm_mul_ps() { struct THelper { static float Call(const float op1, const float op2) { @@ -942,15 +942,15 @@ void TSSEEmulTest::Test_mm_mul_ps() { Test_mm_dualop<float, 4, WrapF(_mm_mul_ps), THelper, float32x4_t, __m128>(); } -void TSSEEmulTest::Test_mm_mul_pd() { - struct THelper { - static double Call(const double op1, const double op2) { - return op1 * op2; - } - }; - Test_mm_dualop<double, 2, WrapD(_mm_mul_pd), THelper, float64x2_t, __m128d>(); -} - +void TSSEEmulTest::Test_mm_mul_pd() { + struct THelper { + static double Call(const double op1, const double op2) { + return op1 * op2; + } + }; + Test_mm_dualop<double, 2, WrapD(_mm_mul_pd), THelper, float64x2_t, __m128d>(); +} + void TSSEEmulTest::Test_mm_div_ps() { struct THelper { static float Call(const float op1, const float op2) { @@ -960,15 +960,15 @@ void TSSEEmulTest::Test_mm_div_ps() { Test_mm_dualop<float, 4, WrapF(_mm_div_ps), THelper, float32x4_t, __m128>(); } -void TSSEEmulTest::Test_mm_div_pd() { - struct THelper { - static double Call(const double op1, const double op2) { - return op1 / op2; - } - }; - Test_mm_dualop<double, 2, WrapD(_mm_div_pd), THelper, float64x2_t, __m128d>(); -} - +void TSSEEmulTest::Test_mm_div_pd() { + struct THelper { + static double Call(const double op1, const double op2) { + return op1 / op2; + } + }; + Test_mm_dualop<double, 2, WrapD(_mm_div_pd), THelper, float64x2_t, __m128d>(); +} + void TSSEEmulTest::Test_mm_max_ps() { struct THelper { static float Call(const float op1, const float op2) { @@ -1119,8 +1119,8 @@ void TSSEEmulTest::Test_mm_dualcmp() { for (unsigned i = 0; i < elemCount; ++i) { /* memcmp is for compare to invalid floats in results */ - const TElem value = TQType<TOp>::As(result)[i]; - UNIT_ASSERT(memcmp(&(procData[i]), &value, sizeof(TElem)) == 0); + const TElem value = TQType<TOp>::As(result)[i]; + UNIT_ASSERT(memcmp(&(procData[i]), &value, sizeof(TElem)) == 0); } } @@ -1289,10 +1289,10 @@ void TSSEEmulTest::Test_mm_set1_ps() { Test_mm_setter_epiXX<float, 4, WrapF(_mm_set1_ps), float32x4_t, __m128>(); } -void TSSEEmulTest::Test_mm_set_ps1() { - Test_mm_setter_epiXX<float, 4, WrapF(_mm_set_ps1), float32x4_t, __m128>(); -} - +void TSSEEmulTest::Test_mm_set_ps1() { + Test_mm_setter_epiXX<float, 4, WrapF(_mm_set_ps1), float32x4_t, __m128>(); +} + void TSSEEmulTest::Test_mm_setzero_si128() { __m128i value = _mm_setzero_si128(); for (unsigned i = 0; i < 4; ++i) @@ -1305,12 +1305,12 @@ void TSSEEmulTest::Test_mm_setzero_ps() { UNIT_ASSERT_EQUAL(0.0, TQType<float32x4_t>::As(value)[i]); } -void TSSEEmulTest::Test_mm_setzero_pd() { - __m128d value = _mm_setzero_pd(); - for (unsigned i = 0; i < 2; ++i) - UNIT_ASSERT_EQUAL(0.0, TQType<float64x2_t>::As(value)[i]); -} - +void TSSEEmulTest::Test_mm_setzero_pd() { + __m128d value = _mm_setzero_pd(); + for (unsigned i = 0; i < 2; ++i) + UNIT_ASSERT_EQUAL(0.0, TQType<float64x2_t>::As(value)[i]); +} + void TSSEEmulTest::Test_mm_loadl_epi64() { char data[64] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1404,16 +1404,16 @@ void TSSEEmulTest::Test_mm_movemask_ps() { UNIT_ASSERT_EQUAL(result, verify); } -void TSSEEmulTest::Test_mm_movemask_ps_2() { - char data[16] = { - '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', - '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF'}; - __m128 value = _mm_loadu_ps((float*)&data); - - int result = _mm_movemask_ps(value); - UNIT_ASSERT_EQUAL(result, 0xf); -} - +void TSSEEmulTest::Test_mm_movemask_ps_2() { + char data[16] = { + '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', + '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF'}; + __m128 value = _mm_loadu_ps((float*)&data); + + int result = _mm_movemask_ps(value); + UNIT_ASSERT_EQUAL(result, 0xf); +} + void TSSEEmulTest::Test_mm_cvtsi128_si32() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1427,19 +1427,19 @@ void TSSEEmulTest::Test_mm_cvtsi128_si32() { UNIT_ASSERT_EQUAL(result, verify); } -void TSSEEmulTest::Test_mm_cvtsi128_si64() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - __m128i value = _mm_loadu_si128((__m128i*)&data); - - i64 result = _mm_cvtsi128_si64(value); - i64* datap = reinterpret_cast<i64*>(&data); - i64 verify = datap[0]; - - UNIT_ASSERT_EQUAL(result, verify); -} - +void TSSEEmulTest::Test_mm_cvtsi128_si64() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + __m128i value = _mm_loadu_si128((__m128i*)&data); + + i64 result = _mm_cvtsi128_si64(value); + i64* datap = reinterpret_cast<i64*>(&data); + i64 verify = datap[0]; + + UNIT_ASSERT_EQUAL(result, verify); +} + void TSSEEmulTest::Test_mm_set_epi16() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1486,21 +1486,21 @@ void TSSEEmulTest::Test_mm_set_ps() { UNIT_ASSERT_EQUAL(buf[1], dataq[1]); } -void TSSEEmulTest::Test_mm_set_pd() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - double* dataw = reinterpret_cast<double*>(&data); - ui64* dataq = reinterpret_cast<ui64*>(&data); - - __m128d result = _mm_set_pd(dataw[1], dataw[0]); - ui64 buf[2] = {53, 81}; - _mm_storeu_pd((double*)&buf, result); - - UNIT_ASSERT_EQUAL(buf[0], dataq[0]); - UNIT_ASSERT_EQUAL(buf[1], dataq[1]); -} - +void TSSEEmulTest::Test_mm_set_pd() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + double* dataw = reinterpret_cast<double*>(&data); + ui64* dataq = reinterpret_cast<ui64*>(&data); + + __m128d result = _mm_set_pd(dataw[1], dataw[0]); + ui64 buf[2] = {53, 81}; + _mm_storeu_pd((double*)&buf, result); + + UNIT_ASSERT_EQUAL(buf[0], dataq[0]); + UNIT_ASSERT_EQUAL(buf[1], dataq[1]); +} + void TSSEEmulTest::Test_mm_cvtsi32_si128() { char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1517,20 +1517,20 @@ void TSSEEmulTest::Test_mm_cvtsi32_si128() { UNIT_ASSERT_EQUAL(buf[3], 0); } -void TSSEEmulTest::Test_mm_cvtsi64_si128() { - char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - i64* dataw = reinterpret_cast<i64*>(&data); - - __m128i result = _mm_cvtsi64_si128(dataw[0]); - i64 buf[2] = {7, 8}; - _mm_storeu_si128((__m128i*)&buf, result); - - UNIT_ASSERT_EQUAL(buf[0], dataw[0]); - UNIT_ASSERT_EQUAL(buf[1], 0); -} - +void TSSEEmulTest::Test_mm_cvtsi64_si128() { + char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + i64* dataw = reinterpret_cast<i64*>(&data); + + __m128i result = _mm_cvtsi64_si128(dataw[0]); + i64 buf[2] = {7, 8}; + _mm_storeu_si128((__m128i*)&buf, result); + + UNIT_ASSERT_EQUAL(buf[0], dataw[0]); + UNIT_ASSERT_EQUAL(buf[1], 0); +} + template <typename TElem, typename TNarrow, unsigned elemCount, typename TFunc> void TSSEEmulTest::Test_mm_packs_epiXX() { char data[32] = { @@ -1598,8 +1598,8 @@ void TSSEEmulTest::Test_mm_extract_epi16() { alignas(16) char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - const ui16* dataw = reinterpret_cast<const ui16*>(&data); - const __m128i value = _mm_loadu_si128((__m128i*)&data); + const ui16* dataw = reinterpret_cast<const ui16*>(&data); + const __m128i value = _mm_loadu_si128((__m128i*)&data); UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0])); UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 1)), int(dataw[1])); @@ -1725,7 +1725,7 @@ void TSSEEmulTest::Test_mm_cvtps_epi32() { } }; Test_mm_convertop<float, i32, 4, __m128, __m128i, - THelper, T_mm_CallWrapper<__m128i, decltype(_mm_cvtps_epi32), _mm_cvtps_epi32>, int32x4_t>(); + THelper, T_mm_CallWrapper<__m128i, decltype(_mm_cvtps_epi32), _mm_cvtps_epi32>, int32x4_t>(); }; void TSSEEmulTest::Test_mm_cvttps_epi32() { @@ -1789,68 +1789,68 @@ void TSSEEmulTest::Test_mm_mul_epu32() { UNIT_ASSERT_EQUAL(mul0, TQType<uint64x2_t>::As(result)[0]); UNIT_ASSERT_EQUAL(mul1, TQType<uint64x2_t>::As(result)[1]); } - -void TSSEEmulTest::Test_mm_cmpunord_ps() { - alignas(16) float valuesBits[4] = {1.f, 2.f, 3.f, 4.f}; - alignas(16) float values2Bits[4] = {5.f, 6.f, 7.f, 8.f}; - - alignas(16) char allfs[16] = { - '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', + +void TSSEEmulTest::Test_mm_cmpunord_ps() { + alignas(16) float valuesBits[4] = {1.f, 2.f, 3.f, 4.f}; + alignas(16) float values2Bits[4] = {5.f, 6.f, 7.f, 8.f}; + + alignas(16) char allfs[16] = { + '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff' }; - - alignas(16) char allzeroes[16] = { - '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + + alignas(16) char allzeroes[16] = { + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00' }; - - const __m128 qnan = _mm_set_ps1(std::numeric_limits<float>::quiet_NaN()); - const __m128 snan = _mm_set_ps1(std::numeric_limits<float>::signaling_NaN()); + + const __m128 qnan = _mm_set_ps1(std::numeric_limits<float>::quiet_NaN()); + const __m128 snan = _mm_set_ps1(std::numeric_limits<float>::signaling_NaN()); const __m128 values = _mm_loadu_ps((const float*) valuesBits); const __m128 values2 = _mm_loadu_ps((const float*) values2Bits); - - const __m128 mask1 = _mm_cmpunord_ps(qnan, qnan); - UNIT_ASSERT_EQUAL(::memcmp(&mask1, &allfs, sizeof(allfs)), 0); - - const __m128 mask2 = _mm_cmpunord_ps(values, values); - UNIT_ASSERT_EQUAL(::memcmp(&mask2, &allzeroes, sizeof(allzeroes)), 0); - - const __m128 mask3 = _mm_cmpunord_ps(snan, snan); - UNIT_ASSERT_EQUAL(::memcmp(&mask3, &allfs, sizeof(allfs)), 0); - - const __m128 mask4 = _mm_cmpunord_ps(qnan, values); - UNIT_ASSERT_EQUAL(::memcmp(&mask4, &allfs, sizeof(allfs)), 0); - - const __m128 mask5 = _mm_cmpunord_ps(snan, values); - UNIT_ASSERT_EQUAL(::memcmp(&mask5, &allfs, sizeof(allfs)), 0); - - const __m128 mask6 = _mm_cmpunord_ps(qnan, snan); - UNIT_ASSERT_EQUAL(::memcmp(&mask6, &allfs, sizeof(allfs)), 0); - - const __m128 mask7 = _mm_cmpunord_ps(values, values2); - UNIT_ASSERT_EQUAL(::memcmp(&mask7, &allzeroes, sizeof(allzeroes)), 0); -} - -void TSSEEmulTest::Test_mm_store_ss() { - alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f}; - const __m128 value = _mm_loadu_ps(valueBits); - float res = std::numeric_limits<float>::signaling_NaN(); - _mm_store_ss(&res, value); - UNIT_ASSERT_EQUAL(res, 1.f); -} - -void TSSEEmulTest::Test_mm_store_ps() { - alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f}; - const __m128 value = _mm_loadu_ps(valueBits); - float res[4] = {0.f}; - _mm_storeu_ps(res, value); - UNIT_ASSERT_EQUAL(res[0], 1.f); - UNIT_ASSERT_EQUAL(res[1], 2.f); - UNIT_ASSERT_EQUAL(res[2], 3.f); - UNIT_ASSERT_EQUAL(res[3], 4.f); -} - -void TSSEEmulTest::Test_mm_storeu_pd() { + + const __m128 mask1 = _mm_cmpunord_ps(qnan, qnan); + UNIT_ASSERT_EQUAL(::memcmp(&mask1, &allfs, sizeof(allfs)), 0); + + const __m128 mask2 = _mm_cmpunord_ps(values, values); + UNIT_ASSERT_EQUAL(::memcmp(&mask2, &allzeroes, sizeof(allzeroes)), 0); + + const __m128 mask3 = _mm_cmpunord_ps(snan, snan); + UNIT_ASSERT_EQUAL(::memcmp(&mask3, &allfs, sizeof(allfs)), 0); + + const __m128 mask4 = _mm_cmpunord_ps(qnan, values); + UNIT_ASSERT_EQUAL(::memcmp(&mask4, &allfs, sizeof(allfs)), 0); + + const __m128 mask5 = _mm_cmpunord_ps(snan, values); + UNIT_ASSERT_EQUAL(::memcmp(&mask5, &allfs, sizeof(allfs)), 0); + + const __m128 mask6 = _mm_cmpunord_ps(qnan, snan); + UNIT_ASSERT_EQUAL(::memcmp(&mask6, &allfs, sizeof(allfs)), 0); + + const __m128 mask7 = _mm_cmpunord_ps(values, values2); + UNIT_ASSERT_EQUAL(::memcmp(&mask7, &allzeroes, sizeof(allzeroes)), 0); +} + +void TSSEEmulTest::Test_mm_store_ss() { + alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f}; + const __m128 value = _mm_loadu_ps(valueBits); + float res = std::numeric_limits<float>::signaling_NaN(); + _mm_store_ss(&res, value); + UNIT_ASSERT_EQUAL(res, 1.f); +} + +void TSSEEmulTest::Test_mm_store_ps() { + alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f}; + const __m128 value = _mm_loadu_ps(valueBits); + float res[4] = {0.f}; + _mm_storeu_ps(res, value); + UNIT_ASSERT_EQUAL(res[0], 1.f); + UNIT_ASSERT_EQUAL(res[1], 2.f); + UNIT_ASSERT_EQUAL(res[2], 3.f); + UNIT_ASSERT_EQUAL(res[3], 4.f); +} + +void TSSEEmulTest::Test_mm_storeu_pd() { alignas(16) const double valueBits[4] = {1., 2., 3., 4.}; for (size_t i = 0; i != 3; ++i) { const __m128d value = _mm_loadu_pd(&valueBits[i]); @@ -1862,43 +1862,43 @@ void TSSEEmulTest::Test_mm_storeu_pd() { } } } -} - -void TSSEEmulTest::Test_mm_andnot_ps() { - alignas(16) const char firstBits[16] = { - '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', +} + +void TSSEEmulTest::Test_mm_andnot_ps() { + alignas(16) const char firstBits[16] = { + '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff' }; - - alignas(16) const char secondBits[16] = { - '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', + + alignas(16) const char secondBits[16] = { + '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff' }; - - alignas(16) const char resBits[16] = { - '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00', + + alignas(16) const char resBits[16] = { + '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00' }; - + const __m128 value1 = _mm_loadu_ps((const float*) firstBits); const __m128 value2 = _mm_loadu_ps((const float*) secondBits); - const __m128 res = _mm_andnot_ps(value1, value2); - - UNIT_ASSERT_EQUAL(::memcmp(&res, resBits, sizeof(resBits)), 0); -} - -void TSSEEmulTest::Test_mm_shuffle_ps() { - alignas(16) const float first[4] = {1.f, 2.f, 3.f, 4.f}; - alignas(16) const float second[4] = {5.f, 6.f, 7.f, 8.f}; - alignas(16) const float etalon[4] = {3.f, 4.f, 5.f, 6.f}; - - const __m128 value1 = _mm_loadu_ps(first); - const __m128 value2 = _mm_loadu_ps(second); - const __m128 res = _mm_shuffle_ps(value1, value2, _MM_SHUFFLE(1, 0, 3, 2)); - - UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); -} - + const __m128 res = _mm_andnot_ps(value1, value2); + + UNIT_ASSERT_EQUAL(::memcmp(&res, resBits, sizeof(resBits)), 0); +} + +void TSSEEmulTest::Test_mm_shuffle_ps() { + alignas(16) const float first[4] = {1.f, 2.f, 3.f, 4.f}; + alignas(16) const float second[4] = {5.f, 6.f, 7.f, 8.f}; + alignas(16) const float etalon[4] = {3.f, 4.f, 5.f, 6.f}; + + const __m128 value1 = _mm_loadu_ps(first); + const __m128 value2 = _mm_loadu_ps(second); + const __m128 res = _mm_shuffle_ps(value1, value2, _MM_SHUFFLE(1, 0, 3, 2)); + + UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); +} + void TSSEEmulTest::Test_mm_shuffle_pd() { const double first[2] = {1.3, 2.3}; const double second[2] = {5.3, 6.3}; @@ -1968,28 +1968,28 @@ void TSSEEmulTest::Test_mm_loadh_pd() { UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0); } -void TSSEEmulTest::Test_mm_or_ps() { - alignas(16) const char bytes1[16] = { - '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', +void TSSEEmulTest::Test_mm_or_ps() { + alignas(16) const char bytes1[16] = { + '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff' }; - - alignas(16) const char bytes2[16] = { - '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', + + alignas(16) const char bytes2[16] = { + '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff' }; - - alignas(16) const char etalon[16] = { - '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff', + + alignas(16) const char etalon[16] = { + '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff' }; - + const __m128 value1 = _mm_loadu_ps((const float*) bytes1); const __m128 value2 = _mm_loadu_ps((const float*) bytes2); - const __m128 res = _mm_or_ps(value1, value2); - - UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); -} + const __m128 res = _mm_or_ps(value1, value2); + + UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); +} void TSSEEmulTest::Test_mm_loadu_pd() { alignas(16) double stub[4] = { |