diff options
author | danlark <danlark@yandex-team.ru> | 2022-02-10 16:46:10 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:10 +0300 |
commit | baa58daefa91fde4b4769facdbd2903763b9c6a8 (patch) | |
tree | 1a2c5ffcf89eb53ecd79dbc9bc0a195c27404d0c /library/cpp/sse | |
parent | 3426a9bc7f169ae9da54cef557ad2a33f6e8eee0 (diff) | |
download | ydb-baa58daefa91fde4b4769facdbd2903763b9c6a8.tar.gz |
Restoring authorship annotation for <danlark@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/sse')
-rw-r--r-- | library/cpp/sse/powerpc.h | 1662 | ||||
-rw-r--r-- | library/cpp/sse/sse.cpp | 2 | ||||
-rw-r--r-- | library/cpp/sse/sse.h | 6 | ||||
-rw-r--r-- | library/cpp/sse/sse2neon.h | 406 | ||||
-rw-r--r-- | library/cpp/sse/ut/test.cpp | 510 | ||||
-rw-r--r-- | library/cpp/sse/ut/ya.make | 10 | ||||
-rw-r--r-- | library/cpp/sse/ya.make | 10 |
7 files changed, 1303 insertions, 1303 deletions
diff --git a/library/cpp/sse/powerpc.h b/library/cpp/sse/powerpc.h index e717bf9f9c..82fc011fa5 100644 --- a/library/cpp/sse/powerpc.h +++ b/library/cpp/sse/powerpc.h @@ -10,991 +10,991 @@ #if !defined(_ppc64_) #error "This header is for PowerPC (ppc64) platform only." \ - "Include sse.h instead of including this header directly." + "Include sse.h instead of including this header directly." #endif #include <util/system/types.h> -#include <util/system/compiler.h> +#include <util/system/compiler.h> #include <altivec.h> -typedef __attribute__((__aligned__(8))) unsigned long long __m64; -typedef __attribute__((__aligned__(16), __may_alias__)) vector float __m128; -typedef __attribute__((__aligned__(16), __may_alias__)) vector unsigned char __m128i; -typedef __attribute__((__aligned__(16), __may_alias__)) vector double __m128d; - -using __v2df = __vector double; -using __v2di = __vector long long; -using __v2du = __vector unsigned long long; -using __v4si = __vector int; -using __v4su = __vector unsigned int; -using __v8hi = __vector short; -using __v8hu = __vector unsigned short; -using __v16qi = __vector signed char; -using __v16qu = __vector unsigned char; -using __v4sf = __vector float; - -enum _mm_hint -{ - /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ - _MM_HINT_ET0 = 7, - _MM_HINT_ET1 = 6, - _MM_HINT_T0 = 3, - _MM_HINT_T1 = 2, - _MM_HINT_T2 = 1, - _MM_HINT_NTA = 0 -}; - -#define _MM_SHUFFLE(a, b, c, d) ((signed char)(a * 64 + b * 16 + c * 4 + d)) - +typedef __attribute__((__aligned__(8))) unsigned long long __m64; +typedef __attribute__((__aligned__(16), __may_alias__)) vector float __m128; +typedef __attribute__((__aligned__(16), __may_alias__)) vector unsigned char __m128i; +typedef __attribute__((__aligned__(16), __may_alias__)) vector double __m128d; + +using __v2df = __vector double; +using __v2di = __vector long long; +using __v2du = __vector unsigned long long; +using __v4si = __vector int; +using __v4su = __vector unsigned int; +using __v8hi = __vector short; +using __v8hu = __vector unsigned short; +using __v16qi = __vector signed char; +using __v16qu = __vector unsigned char; +using __v4sf = __vector float; + +enum _mm_hint +{ + /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ + _MM_HINT_ET0 = 7, + _MM_HINT_ET1 = 6, + _MM_HINT_T0 = 3, + _MM_HINT_T1 = 2, + _MM_HINT_T2 = 1, + _MM_HINT_NTA = 0 +}; + +#define _MM_SHUFFLE(a, b, c, d) ((signed char)(a * 64 + b * 16 + c * 4 + d)) + /// Functions that work with floats. -Y_FORCE_INLINE __m128 _mm_setzero_ps() { - return (__m128){0.0f, 0.0f, 0.0f, 0.0f}; +Y_FORCE_INLINE __m128 _mm_setzero_ps() { + return (__m128){0.0f, 0.0f, 0.0f, 0.0f}; }; -Y_FORCE_INLINE __m128d _mm_setzero_pd() { - return (__m128d)vec_splats((double)0); +Y_FORCE_INLINE __m128d _mm_setzero_pd() { + return (__m128d)vec_splats((double)0); } -// bug in clang compiler until 7.0.0 inclusive, Y_NO_INLINE is vital/essential -static Y_NO_INLINE __m128 _mm_set1_ps(float f) { - return (vector float)f; +// bug in clang compiler until 7.0.0 inclusive, Y_NO_INLINE is vital/essential +static Y_NO_INLINE __m128 _mm_set1_ps(float f) { + return (vector float)f; } -Y_FORCE_INLINE __m128 _mm_set_ps1(float f) { +Y_FORCE_INLINE __m128 _mm_set_ps1(float f) { return _mm_set1_ps(f); } -Y_FORCE_INLINE __m128 _mm_set_ps(float v3, float v2, float v1, float v0) { - return (__m128)(__v4sf){v0, v1, v2, v3}; +Y_FORCE_INLINE __m128 _mm_set_ps(float v3, float v2, float v1, float v0) { + return (__m128)(__v4sf){v0, v1, v2, v3}; } -Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { - return (__m128d){d0, d1}; +Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { + return (__m128d){d0, d1}; } -Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* p) { - return vec_vsx_ld(0, p); +Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* p) { + return vec_vsx_ld(0, p); } -Y_FORCE_INLINE __m128 _mm_load_ps(const float* p) { - return (__m128)vec_ld(0, (vector float*)p); +Y_FORCE_INLINE __m128 _mm_load_ps(const float* p) { + return (__m128)vec_ld(0, (vector float*)p); } -Y_FORCE_INLINE __m128 _mm_loadu_pd(const double* d) { - return vec_vsx_ld(0, d); +Y_FORCE_INLINE __m128 _mm_loadu_pd(const double* d) { + return vec_vsx_ld(0, d); } -Y_FORCE_INLINE void _mm_storeu_ps(float* p, __m128 a) { - *(__m128*)p = a; +Y_FORCE_INLINE void _mm_storeu_ps(float* p, __m128 a) { + *(__m128*)p = a; } -Y_FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) { - return (__m128)vec_xor((__v4sf)a, (__v4sf)b); -} - -Y_FORCE_INLINE __m128 _mm_xor_pd(__m128d a, __m128d b) { - return (__m128)vec_xor((__v2df)a, (__v2df)b); -} - -Y_FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) { - return (__m128)((__v4sf)a + (__v4sf)b); +Y_FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) { + return (__m128)vec_xor((__v4sf)a, (__v4sf)b); } -Y_FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) { - return (__m128d)((__v2df)a + (__v2df)b); +Y_FORCE_INLINE __m128 _mm_xor_pd(__m128d a, __m128d b) { + return (__m128)vec_xor((__v2df)a, (__v2df)b); } -Y_FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) { - return (__m128)((__v4sf)a - (__v4sf)b); +Y_FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) { + return (__m128)((__v4sf)a + (__v4sf)b); } -Y_FORCE_INLINE __m128d _mm_sub_pd(__m128d a, __m128d b) { - return (__m128d)((__v2df)a - (__v2df)b); +Y_FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) { + return (__m128d)((__v2df)a + (__v2df)b); } -Y_FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) { - return (__m128)((__v4sf)a * (__v4sf)b); +Y_FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) { + return (__m128)((__v4sf)a - (__v4sf)b); } -Y_FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) { - return (__m128d)((__v2df)a * (__v2df)b); +Y_FORCE_INLINE __m128d _mm_sub_pd(__m128d a, __m128d b) { + return (__m128d)((__v2df)a - (__v2df)b); } -Y_FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) { - return (__m128)((__v4sf)a / (__v4sf)b); +Y_FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) { + return (__m128)((__v4sf)a * (__v4sf)b); } -Y_FORCE_INLINE __m128d _mm_div_pd(__m128d a, __m128d b) { - return (__m128d)((__v2df)a / (__v2df)b); +Y_FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) { + return (__m128d)((__v2df)a * (__v2df)b); } -Y_FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) { - return ((__m128)vec_cmpeq((__v4sf)a, (__v4sf)b)); - ; +Y_FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) { + return (__m128)((__v4sf)a / (__v4sf)b); } -Y_FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) { - return ((__m128)vec_cmpgt((__v4sf)a, (__v4sf)b)); +Y_FORCE_INLINE __m128d _mm_div_pd(__m128d a, __m128d b) { + return (__m128d)((__v2df)a / (__v2df)b); } -Y_FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) { - return (__m128)vec_max((vector float)a, (vector float)b); +Y_FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) { + return ((__m128)vec_cmpeq((__v4sf)a, (__v4sf)b)); + ; } -Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) { - return (__m128i)vec_max((__v16qu)a, (__v16qu)b); -} - -Y_FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) { - return (__m128)vec_min((vector float)a, (vector float)b); +Y_FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) { + return ((__m128)vec_cmpgt((__v4sf)a, (__v4sf)b)); } -Y_FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) { - return ((__m128)vec_and((__v4sf)a, (__v4sf)b)); +Y_FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) { + return (__m128)vec_max((vector float)a, (vector float)b); } -Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { - return vec_and((__v2df)a, (__v2df)b); -} - -Y_FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 a) { - return vec_rsqrte(a); +Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) { + return (__m128i)vec_max((__v16qu)a, (__v16qu)b); } -Y_FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 a) { - __m128 a1, c; - const vector unsigned int mask = {0xffffffff, 0, 0, 0}; - a1 = vec_splat(a, 0); - c = vec_rsqrte(a1); - return (vec_sel((vector float)a, c, mask)); +Y_FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) { + return (__m128)vec_min((vector float)a, (vector float)b); } -Y_FORCE_INLINE int _mm_movemask_ps(__m128 a) { - __vector unsigned long long result; - const __vector unsigned int perm_mask = - { -#ifdef __LITTLE_ENDIAN__ - 0x00204060, 0x80808080, 0x80808080, 0x80808080 -#elif __BIG_ENDIAN__ - 0x80808080, 0x80808080, 0x80808080, 0x00204060 -#endif - }; - - result = (__vector unsigned long long)vec_vbpermq((__vector unsigned char)a, - (__vector unsigned char)perm_mask); - -#ifdef __LITTLE_ENDIAN__ - return result[1]; -#elif __BIG_ENDIAN__ - return result[0]; -#endif +Y_FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) { + return ((__m128)vec_and((__v4sf)a, (__v4sf)b)); } -Y_FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) { - return ((__m128)vec_ctf((__v4si)a, 0)); +Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { + return vec_and((__v2df)a, (__v2df)b); } -Y_FORCE_INLINE float _mm_cvtss_f32(__m128 a) { - return ((__v4sf)a)[0]; +Y_FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 a) { + return vec_rsqrte(a); } -Y_FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 A, __m128 B) { - __vector unsigned int a, b; - __vector unsigned int c, d; - const __vector unsigned int float_exp_mask = - {0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000}; - - a = (__vector unsigned int)vec_abs((__v4sf)A); - b = (__vector unsigned int)vec_abs((__v4sf)B); - c = (__vector unsigned int)vec_cmpgt(a, float_exp_mask); - d = (__vector unsigned int)vec_cmpgt(b, float_exp_mask); - return ((__m128)vec_or(c, d)); +Y_FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 a) { + __m128 a1, c; + const vector unsigned int mask = {0xffffffff, 0, 0, 0}; + a1 = vec_splat(a, 0); + c = vec_rsqrte(a1); + return (vec_sel((vector float)a, c, mask)); } -Y_FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) { - return ((__m128)vec_andc((__v4sf)b, (__v4sf)a)); +Y_FORCE_INLINE int _mm_movemask_ps(__m128 a) { + __vector unsigned long long result; + const __vector unsigned int perm_mask = + { +#ifdef __LITTLE_ENDIAN__ + 0x00204060, 0x80808080, 0x80808080, 0x80808080 +#elif __BIG_ENDIAN__ + 0x80808080, 0x80808080, 0x80808080, 0x00204060 +#endif + }; + + result = (__vector unsigned long long)vec_vbpermq((__vector unsigned char)a, + (__vector unsigned char)perm_mask); + +#ifdef __LITTLE_ENDIAN__ + return result[1]; +#elif __BIG_ENDIAN__ + return result[0]; +#endif } -Y_FORCE_INLINE __m128 _mm_or_ps(__m128 a, __m128 b) { - return ((__m128)vec_or((__v4sf)a, (__v4sf)b)); +Y_FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) { + return ((__m128)vec_ctf((__v4si)a, 0)); } -Y_FORCE_INLINE void _mm_store_ss(float* p, __m128 a) { - *p = ((__v4sf)a)[0]; +Y_FORCE_INLINE float _mm_cvtss_f32(__m128 a) { + return ((__v4sf)a)[0]; } -Y_FORCE_INLINE void _mm_store_ps(float* p, __m128 a) { - vec_st(a, 0, p); +Y_FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 A, __m128 B) { + __vector unsigned int a, b; + __vector unsigned int c, d; + const __vector unsigned int float_exp_mask = + {0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000}; + + a = (__vector unsigned int)vec_abs((__v4sf)A); + b = (__vector unsigned int)vec_abs((__v4sf)B); + c = (__vector unsigned int)vec_cmpgt(a, float_exp_mask); + d = (__vector unsigned int)vec_cmpgt(b, float_exp_mask); + return ((__m128)vec_or(c, d)); +} + +Y_FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) { + return ((__m128)vec_andc((__v4sf)b, (__v4sf)a)); } -Y_FORCE_INLINE void _mm_storeu_pd(double* p, __m128d a) { - *(__m128d*)p = a; -} - -Y_FORCE_INLINE void _mm_store_pd(double* p, __m128d a) { - vec_st((vector unsigned char)a, 0, (vector unsigned char*)p); -} - -Y_FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, long shuff) { - unsigned long element_selector_10 = shuff & 0x03; - unsigned long element_selector_32 = (shuff >> 2) & 0x03; - unsigned long element_selector_54 = (shuff >> 4) & 0x03; - unsigned long element_selector_76 = (shuff >> 6) & 0x03; - const unsigned int permute_selectors[4] = - { -#ifdef __LITTLE_ENDIAN__ - 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C -#elif __BIG_ENDIAN__ - 0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203 -#endif - }; - __vector unsigned int t; - -#ifdef __LITTLE_ENDIAN__ - t[0] = permute_selectors[element_selector_10]; - t[1] = permute_selectors[element_selector_32]; - t[2] = permute_selectors[element_selector_54] + 0x10101010; - t[3] = permute_selectors[element_selector_76] + 0x10101010; -#elif __BIG_ENDIAN__ - t[3] = permute_selectors[element_selector_10] + 0x10101010; - t[2] = permute_selectors[element_selector_32] + 0x10101010; - t[1] = permute_selectors[element_selector_54]; - t[0] = permute_selectors[element_selector_76]; -#endif - return vec_perm((__v4sf)a, (__v4sf)b, (__vector unsigned char)t); -} - -Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, const int mask) { - __vector double result; - const int litmsk = mask & 0x3; - - if (litmsk == 0) - result = vec_mergeh(a, b); - else if (litmsk == 1) - result = vec_xxpermdi(a, b, 2); - else if (litmsk == 2) - result = vec_xxpermdi(a, b, 1); - else - result = vec_mergel(a, b); - return result; -} - -Y_FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) { - vector float rounded; - __v4si result; - - rounded = vec_rint((vector float)a); - result = vec_cts(rounded, 0); - return (__m128i)result; +Y_FORCE_INLINE __m128 _mm_or_ps(__m128 a, __m128 b) { + return ((__m128)vec_or((__v4sf)a, (__v4sf)b)); +} + +Y_FORCE_INLINE void _mm_store_ss(float* p, __m128 a) { + *p = ((__v4sf)a)[0]; +} + +Y_FORCE_INLINE void _mm_store_ps(float* p, __m128 a) { + vec_st(a, 0, p); +} + +Y_FORCE_INLINE void _mm_storeu_pd(double* p, __m128d a) { + *(__m128d*)p = a; +} + +Y_FORCE_INLINE void _mm_store_pd(double* p, __m128d a) { + vec_st((vector unsigned char)a, 0, (vector unsigned char*)p); +} + +Y_FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, long shuff) { + unsigned long element_selector_10 = shuff & 0x03; + unsigned long element_selector_32 = (shuff >> 2) & 0x03; + unsigned long element_selector_54 = (shuff >> 4) & 0x03; + unsigned long element_selector_76 = (shuff >> 6) & 0x03; + const unsigned int permute_selectors[4] = + { +#ifdef __LITTLE_ENDIAN__ + 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C +#elif __BIG_ENDIAN__ + 0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203 +#endif + }; + __vector unsigned int t; + +#ifdef __LITTLE_ENDIAN__ + t[0] = permute_selectors[element_selector_10]; + t[1] = permute_selectors[element_selector_32]; + t[2] = permute_selectors[element_selector_54] + 0x10101010; + t[3] = permute_selectors[element_selector_76] + 0x10101010; +#elif __BIG_ENDIAN__ + t[3] = permute_selectors[element_selector_10] + 0x10101010; + t[2] = permute_selectors[element_selector_32] + 0x10101010; + t[1] = permute_selectors[element_selector_54]; + t[0] = permute_selectors[element_selector_76]; +#endif + return vec_perm((__v4sf)a, (__v4sf)b, (__vector unsigned char)t); +} + +Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, const int mask) { + __vector double result; + const int litmsk = mask & 0x3; + + if (litmsk == 0) + result = vec_mergeh(a, b); + else if (litmsk == 1) + result = vec_xxpermdi(a, b, 2); + else if (litmsk == 2) + result = vec_xxpermdi(a, b, 1); + else + result = vec_mergel(a, b); + return result; +} + +Y_FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) { + vector float rounded; + __v4si result; + + rounded = vec_rint((vector float)a); + result = vec_cts(rounded, 0); + return (__m128i)result; } /// Functions that work with integers. -Y_FORCE_INLINE int _mm_movemask_epi8(__m128i a) { - __vector unsigned long long result; - const __vector unsigned char perm_mask = - { -#ifdef __LITTLE_ENDIAN__ - 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, - 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00 -#elif __BIG_ENDIAN__ - 0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, - 0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78 -#endif - }; - - result = (__vector unsigned long long)vec_vbpermq((__vector unsigned char)a, - (__vector unsigned char)perm_mask); - -#ifdef __LITTLE_ENDIAN__ - return result[1]; -#elif __BIG_ENDIAN__ - return result[0]; -#endif -} - -Y_FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) { - __v4si result; - - result = vec_cts((__v4sf)a, 0); - return (__m128i)result; -} - -#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ - do { \ - __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ - __v4sf __t0 = vec_vmrghw((vector unsigned int)__r0, (vector unsigned int)__r1); \ - __v4sf __t1 = vec_vmrghw((vector unsigned int)__r2, (vector unsigned int)__r3); \ - __v4sf __t2 = vec_vmrglw((vector unsigned int)__r0, (vector unsigned int)__r1); \ - __v4sf __t3 = vec_vmrglw((vector unsigned int)__r2, (vector unsigned int)__r3); \ - (row0) = (__v4sf)vec_mergeh((vector long long)__t0, \ - (vector long long)__t1); \ - (row1) = (__v4sf)vec_mergel((vector long long)__t0, \ - (vector long long)__t1); \ - (row2) = (__v4sf)vec_mergeh((vector long long)__t2, \ - (vector long long)__t3); \ - (row3) = (__v4sf)vec_mergel((vector long long)__t2, \ - (vector long long)__t3); \ - } while (0) - -Y_FORCE_INLINE __m128i _mm_or_si128(__m128i a, __m128i b) { - return (__m128i)vec_or((__v2di)a, (__v2di)b); -} - -Y_FORCE_INLINE __m128i _mm_and_si128(__m128i a, __m128i b) { - return (__m128i)vec_and((__v2di)a, (__v2di)b); -} - -Y_FORCE_INLINE __m128i _mm_andnot_si128(__m128i a, __m128i b) { - return (__m128i)vec_andc((__v2di)b, (__v2di)a); -} - -Y_FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) { - return (__m128i)vec_xor((__v2di)a, (__v2di)b); -} - -Y_FORCE_INLINE __m128i _mm_setzero_si128() { - return (__m128i)(__v4si){0, 0, 0, 0}; -} - -Y_FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i op1, long op2) { - unsigned long element_selector_10 = op2 & 0x03; - unsigned long element_selector_32 = (op2 >> 2) & 0x03; - unsigned long element_selector_54 = (op2 >> 4) & 0x03; - unsigned long element_selector_76 = (op2 >> 6) & 0x03; - const unsigned int permute_selectors[4] = - { -#ifdef __LITTLE_ENDIAN__ - 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C -#elif __BIG_ENDIAN__ - 0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203 -#endif - }; - __v4su t; - -#ifdef __LITTLE_ENDIAN__ - t[0] = permute_selectors[element_selector_10]; - t[1] = permute_selectors[element_selector_32]; - t[2] = permute_selectors[element_selector_54] + 0x10101010; - t[3] = permute_selectors[element_selector_76] + 0x10101010; -#elif __BIG_ENDIAN__ - t[3] = permute_selectors[element_selector_10] + 0x10101010; - t[2] = permute_selectors[element_selector_32] + 0x10101010; - t[1] = permute_selectors[element_selector_54]; - t[0] = permute_selectors[element_selector_76]; -#endif - return (__m128i)vec_perm((__v4si)op1, (__v4si)op1, (__vector unsigned char)t); -} - -Y_FORCE_INLINE int _mm_extract_epi16(__m128i a, int imm) { - return (unsigned short)((__v8hi)a)[imm & 7]; -} - -Y_FORCE_INLINE int _mm_extract_epi8(__m128i a, int imm) { - return (unsigned char)((__v16qi)a)[imm & 15]; -} - -Y_FORCE_INLINE int _mm_extract_epi32(__m128i a, int imm) { - return ((__v4si)a)[imm & 3]; -} - -Y_FORCE_INLINE long long _mm_extract_epi64(__m128i a, int imm) { - return ((__v2di)a)[imm & 1]; -} - -Y_FORCE_INLINE int _mm_extract_ps(__m128 a, int imm) { - return ((__v4si)a)[imm & 3]; -} - -Y_FORCE_INLINE __m128i _mm_slli_epi16(__m128i a, int count) { - __v8hu lshift; - __v8hi result = {0, 0, 0, 0, 0, 0, 0, 0}; - - if (count >= 0 && count < 16) { - if (__builtin_constant_p(count)) { - lshift = (__v8hu)vec_splat_s16(count); - } else { - lshift = vec_splats((unsigned short)count); - } - - result = vec_vslh((__v8hi)a, lshift); - } - - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int count) { - __v4su lshift; - __v4si result = {0, 0, 0, 0}; - - if (count >= 0 && count < 32) { - if (__builtin_constant_p(count) && count < 16) { - lshift = (__v4su)vec_splat_s32(count); - } else { - lshift = vec_splats((unsigned int)count); - } - - result = vec_vslw((__v4si)a, lshift); - } - - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int count) { - __v2du lshift; - __v2di result = {0, 0}; - - if (count >= 0 && count < 64) { - if (__builtin_constant_p(count) && count < 16) { - lshift = (__v2du)vec_splat_s32(count); - } else { - lshift = (__v2du)vec_splats((unsigned int)count); - } - - result = vec_sl((__v2di)a, lshift); - } - - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_slli_si128(__m128i a, int imm) { - __v16qu result; - const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - - if (imm < 16) -#ifdef __LITTLE_ENDIAN__ - result = vec_sld((__v16qu)a, zeros, imm); -#elif __BIG_ENDIAN__ - result = vec_sld(zeros, (__v16qu)a, (16 - imm)); -#endif - else - result = zeros; - - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_srli_epi16(__m128i a, int count) { - if ((unsigned long)count >= 16) { - /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ - return (__m128i)vec_splats(0); - } else if (count == 0) { - return a; - } else { - /* The PowerPC Architecture says all shift count fields must contain the same shift count. */ - __v8hi replicated_count; - replicated_count = vec_splats((short)count); - return (__m128i)vec_sr((vector signed short)a, replicated_count); - } -} - -Y_FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, int count) { - if ((unsigned long)count >= 32) { - /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ - return (__m128i)vec_splats(0); - } else if (count == 0) { - return a; - } else { - /* The PowerPC Architecture says all shift count fields must contain the same shift count. */ - __v4si replicated_count; - replicated_count = vec_splats(count); - return (__m128i)vec_sr((vector signed int)a, replicated_count); - } -} - -Y_FORCE_INLINE __m128i _mm_srli_epi64(__m128i a, int count) { - if ((unsigned long)count >= 64) { - /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ - return (__m128i)vec_splats(0); - } else if (count == 0) { - return a; - } else { - /* The PowerPC Architecture says all shift count fields must contain the same shift count. */ - /* On Power7 vec_slo (vslo) does use just the documented bits 121:124. */ - /* On Power7 vec_sll (vsll) uses the lower 3 bits of each byte instead (legal). */ - __v16qu replicated_count; - replicated_count = vec_splats((unsigned char)count); - long long m = 0xFFFFFFFFFFFFFFFFull >> count; - __v2di mask; - mask[0] = m; - mask[1] = m; - return vec_and(vec_srl(vec_sro(a, (__m128i)replicated_count), (__m128i)replicated_count), (__v16qu)mask); - } -} - -Y_FORCE_INLINE __m128i _mm_bsrli_si128(__m128i a, const int __N) { - __v16qu result; - const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - - if (__N < 16) - if (__builtin_constant_p(__N)) - /* Would like to use Vector Shift Left Double by Octet - Immediate here to use the immediate form and avoid - load of __N * 8 value into a separate VR. */ - result = vec_sld(zeros, (__v16qu)a, (16 - __N)); - else { - __v16qu shift = vec_splats((unsigned char)(__N * 8)); - result = vec_sro((__v16qu)a, shift); - } - else - result = zeros; - - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_srli_si128(__m128i a, int imm) { - return _mm_bsrli_si128(a, imm); -} - -Y_FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) { - __v8hu rshift = {15, 15, 15, 15, 15, 15, 15, 15}; - __v8hi result; - if (count < 16) { - if (__builtin_constant_p(count)) { - rshift = (__v8hu)vec_splat_s16(count); - } else { - rshift = vec_splats((unsigned short)count); - } - } - result = vec_vsrah((__v8hi)a, rshift); - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, int count) { +Y_FORCE_INLINE int _mm_movemask_epi8(__m128i a) { + __vector unsigned long long result; + const __vector unsigned char perm_mask = + { +#ifdef __LITTLE_ENDIAN__ + 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, + 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00 +#elif __BIG_ENDIAN__ + 0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, + 0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78 +#endif + }; + + result = (__vector unsigned long long)vec_vbpermq((__vector unsigned char)a, + (__vector unsigned char)perm_mask); + +#ifdef __LITTLE_ENDIAN__ + return result[1]; +#elif __BIG_ENDIAN__ + return result[0]; +#endif +} + +Y_FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) { + __v4si result; + + result = vec_cts((__v4sf)a, 0); + return (__m128i)result; +} + +#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ + do { \ + __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ + __v4sf __t0 = vec_vmrghw((vector unsigned int)__r0, (vector unsigned int)__r1); \ + __v4sf __t1 = vec_vmrghw((vector unsigned int)__r2, (vector unsigned int)__r3); \ + __v4sf __t2 = vec_vmrglw((vector unsigned int)__r0, (vector unsigned int)__r1); \ + __v4sf __t3 = vec_vmrglw((vector unsigned int)__r2, (vector unsigned int)__r3); \ + (row0) = (__v4sf)vec_mergeh((vector long long)__t0, \ + (vector long long)__t1); \ + (row1) = (__v4sf)vec_mergel((vector long long)__t0, \ + (vector long long)__t1); \ + (row2) = (__v4sf)vec_mergeh((vector long long)__t2, \ + (vector long long)__t3); \ + (row3) = (__v4sf)vec_mergel((vector long long)__t2, \ + (vector long long)__t3); \ + } while (0) + +Y_FORCE_INLINE __m128i _mm_or_si128(__m128i a, __m128i b) { + return (__m128i)vec_or((__v2di)a, (__v2di)b); +} + +Y_FORCE_INLINE __m128i _mm_and_si128(__m128i a, __m128i b) { + return (__m128i)vec_and((__v2di)a, (__v2di)b); +} + +Y_FORCE_INLINE __m128i _mm_andnot_si128(__m128i a, __m128i b) { + return (__m128i)vec_andc((__v2di)b, (__v2di)a); +} + +Y_FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) { + return (__m128i)vec_xor((__v2di)a, (__v2di)b); +} + +Y_FORCE_INLINE __m128i _mm_setzero_si128() { + return (__m128i)(__v4si){0, 0, 0, 0}; +} + +Y_FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i op1, long op2) { + unsigned long element_selector_10 = op2 & 0x03; + unsigned long element_selector_32 = (op2 >> 2) & 0x03; + unsigned long element_selector_54 = (op2 >> 4) & 0x03; + unsigned long element_selector_76 = (op2 >> 6) & 0x03; + const unsigned int permute_selectors[4] = + { +#ifdef __LITTLE_ENDIAN__ + 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C +#elif __BIG_ENDIAN__ + 0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203 +#endif + }; + __v4su t; + +#ifdef __LITTLE_ENDIAN__ + t[0] = permute_selectors[element_selector_10]; + t[1] = permute_selectors[element_selector_32]; + t[2] = permute_selectors[element_selector_54] + 0x10101010; + t[3] = permute_selectors[element_selector_76] + 0x10101010; +#elif __BIG_ENDIAN__ + t[3] = permute_selectors[element_selector_10] + 0x10101010; + t[2] = permute_selectors[element_selector_32] + 0x10101010; + t[1] = permute_selectors[element_selector_54]; + t[0] = permute_selectors[element_selector_76]; +#endif + return (__m128i)vec_perm((__v4si)op1, (__v4si)op1, (__vector unsigned char)t); +} + +Y_FORCE_INLINE int _mm_extract_epi16(__m128i a, int imm) { + return (unsigned short)((__v8hi)a)[imm & 7]; +} + +Y_FORCE_INLINE int _mm_extract_epi8(__m128i a, int imm) { + return (unsigned char)((__v16qi)a)[imm & 15]; +} + +Y_FORCE_INLINE int _mm_extract_epi32(__m128i a, int imm) { + return ((__v4si)a)[imm & 3]; +} + +Y_FORCE_INLINE long long _mm_extract_epi64(__m128i a, int imm) { + return ((__v2di)a)[imm & 1]; +} + +Y_FORCE_INLINE int _mm_extract_ps(__m128 a, int imm) { + return ((__v4si)a)[imm & 3]; +} + +Y_FORCE_INLINE __m128i _mm_slli_epi16(__m128i a, int count) { + __v8hu lshift; + __v8hi result = {0, 0, 0, 0, 0, 0, 0, 0}; + + if (count >= 0 && count < 16) { + if (__builtin_constant_p(count)) { + lshift = (__v8hu)vec_splat_s16(count); + } else { + lshift = vec_splats((unsigned short)count); + } + + result = vec_vslh((__v8hi)a, lshift); + } + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int count) { + __v4su lshift; + __v4si result = {0, 0, 0, 0}; + + if (count >= 0 && count < 32) { + if (__builtin_constant_p(count) && count < 16) { + lshift = (__v4su)vec_splat_s32(count); + } else { + lshift = vec_splats((unsigned int)count); + } + + result = vec_vslw((__v4si)a, lshift); + } + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int count) { + __v2du lshift; + __v2di result = {0, 0}; + + if (count >= 0 && count < 64) { + if (__builtin_constant_p(count) && count < 16) { + lshift = (__v2du)vec_splat_s32(count); + } else { + lshift = (__v2du)vec_splats((unsigned int)count); + } + + result = vec_sl((__v2di)a, lshift); + } + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_slli_si128(__m128i a, int imm) { + __v16qu result; + const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + if (imm < 16) +#ifdef __LITTLE_ENDIAN__ + result = vec_sld((__v16qu)a, zeros, imm); +#elif __BIG_ENDIAN__ + result = vec_sld(zeros, (__v16qu)a, (16 - imm)); +#endif + else + result = zeros; + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_srli_epi16(__m128i a, int count) { + if ((unsigned long)count >= 16) { + /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ + return (__m128i)vec_splats(0); + } else if (count == 0) { + return a; + } else { + /* The PowerPC Architecture says all shift count fields must contain the same shift count. */ + __v8hi replicated_count; + replicated_count = vec_splats((short)count); + return (__m128i)vec_sr((vector signed short)a, replicated_count); + } +} + +Y_FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, int count) { + if ((unsigned long)count >= 32) { + /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ + return (__m128i)vec_splats(0); + } else if (count == 0) { + return a; + } else { + /* The PowerPC Architecture says all shift count fields must contain the same shift count. */ + __v4si replicated_count; + replicated_count = vec_splats(count); + return (__m128i)vec_sr((vector signed int)a, replicated_count); + } +} + +Y_FORCE_INLINE __m128i _mm_srli_epi64(__m128i a, int count) { + if ((unsigned long)count >= 64) { + /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */ + return (__m128i)vec_splats(0); + } else if (count == 0) { + return a; + } else { + /* The PowerPC Architecture says all shift count fields must contain the same shift count. */ + /* On Power7 vec_slo (vslo) does use just the documented bits 121:124. */ + /* On Power7 vec_sll (vsll) uses the lower 3 bits of each byte instead (legal). */ + __v16qu replicated_count; + replicated_count = vec_splats((unsigned char)count); + long long m = 0xFFFFFFFFFFFFFFFFull >> count; + __v2di mask; + mask[0] = m; + mask[1] = m; + return vec_and(vec_srl(vec_sro(a, (__m128i)replicated_count), (__m128i)replicated_count), (__v16qu)mask); + } +} + +Y_FORCE_INLINE __m128i _mm_bsrli_si128(__m128i a, const int __N) { + __v16qu result; + const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + if (__N < 16) + if (__builtin_constant_p(__N)) + /* Would like to use Vector Shift Left Double by Octet + Immediate here to use the immediate form and avoid + load of __N * 8 value into a separate VR. */ + result = vec_sld(zeros, (__v16qu)a, (16 - __N)); + else { + __v16qu shift = vec_splats((unsigned char)(__N * 8)); + result = vec_sro((__v16qu)a, shift); + } + else + result = zeros; + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_srli_si128(__m128i a, int imm) { + return _mm_bsrli_si128(a, imm); +} + +Y_FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) { + __v8hu rshift = {15, 15, 15, 15, 15, 15, 15, 15}; + __v8hi result; + if (count < 16) { + if (__builtin_constant_p(count)) { + rshift = (__v8hu)vec_splat_s16(count); + } else { + rshift = vec_splats((unsigned short)count); + } + } + result = vec_vsrah((__v8hi)a, rshift); + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, int count) { // return vec_shiftrightarithmetic4wimmediate(a, count); //!< Failes to work with count >= 32. - __v4su rshift = {31, 31, 31, 31}; - __v4si result; - - if (count < 32) { - if (__builtin_constant_p(count)) { - if (count < 16) { - rshift = (__v4su)vec_splat_s32(count); - } else { - rshift = (__v4su)vec_splats((unsigned int)count); - } - } else { - rshift = vec_splats((unsigned int)count); - } + __v4su rshift = {31, 31, 31, 31}; + __v4si result; + + if (count < 32) { + if (__builtin_constant_p(count)) { + if (count < 16) { + rshift = (__v4su)vec_splat_s32(count); + } else { + rshift = (__v4su)vec_splats((unsigned int)count); + } + } else { + rshift = vec_splats((unsigned int)count); + } } - result = vec_vsraw((__v4si)a, rshift); - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) { - __v8hu lshift, shmask; - const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15}; - __v8hu result; - -#ifdef __LITTLE_ENDIAN__ - lshift = vec_splat((__v8hu)count, 0); -#elif __BIG_ENDIAN__ - lshift = vec_splat((__v8hu)count, 3); -#endif - shmask = vec_cmple(lshift, shmax); - result = vec_vslh((__v8hu)a, lshift); - result = vec_sel(shmask, result, shmask); - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) { - __v4su lshift, shmask; - const __v4su shmax = {32, 32, 32, 32}; - __v4su result; -#ifdef __LITTLE_ENDIAN__ - lshift = vec_splat((__v4su)count, 0); -#elif __BIG_ENDIAN__ - lshift = vec_splat((__v4su)count, 1); -#endif - shmask = vec_cmplt(lshift, shmax); - result = vec_vslw((__v4su)a, lshift); - result = vec_sel(shmask, result, shmask); + result = vec_vsraw((__v4si)a, rshift); + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) { + __v8hu lshift, shmask; + const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15}; + __v8hu result; + +#ifdef __LITTLE_ENDIAN__ + lshift = vec_splat((__v8hu)count, 0); +#elif __BIG_ENDIAN__ + lshift = vec_splat((__v8hu)count, 3); +#endif + shmask = vec_cmple(lshift, shmax); + result = vec_vslh((__v8hu)a, lshift); + result = vec_sel(shmask, result, shmask); + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) { + __v4su lshift, shmask; + const __v4su shmax = {32, 32, 32, 32}; + __v4su result; +#ifdef __LITTLE_ENDIAN__ + lshift = vec_splat((__v4su)count, 0); +#elif __BIG_ENDIAN__ + lshift = vec_splat((__v4su)count, 1); +#endif + shmask = vec_cmplt(lshift, shmax); + result = vec_vslw((__v4su)a, lshift); + result = vec_sel(shmask, result, shmask); + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) { + __v2du lshift, shmask; + const __v2du shmax = {64, 64}; + __v2du result; + + lshift = (__v2du)vec_splat((__v2du)count, 0); + shmask = vec_cmplt(lshift, shmax); + result = vec_sl((__v2du)a, lshift); + result = ((vector long long)shmask & ~(vector long long)shmask) | ((vector long long)result & (vector long long)shmask); + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) { + __v8hu rshift, shmask; + const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15}; + __v8hu result; + +#ifdef __LITTLE_ENDIAN__ + rshift = vec_splat((__v8hu)count, 0); +#elif __BIG_ENDIAN__ + rshift = vec_splat((__v8hu)count, 3); +#endif + shmask = vec_cmple(rshift, shmax); + result = vec_vsrh((__v8hu)a, rshift); + result = vec_sel(shmask, result, shmask); + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) { + __v4su rshift, shmask; + const __v4su shmax = {32, 32, 32, 32}; + __v4su result; + +#ifdef __LITTLE_ENDIAN__ + rshift = vec_splat((__v4su)count, 0); +#elif __BIG_ENDIAN__ + rshift = vec_splat((__v4su)count, 1); +#endif + shmask = vec_cmplt(rshift, shmax); + result = vec_vsrw((__v4su)a, rshift); + result = vec_sel(shmask, result, shmask); + + return (__m128i)result; +} + +Y_FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) { + __v2du rshift, shmask; + const __v2du shmax = {64, 64}; + __v2du result; + + rshift = (__v2du)vec_splat((__v2du)count, 0); + shmask = vec_cmplt(rshift, shmax); + result = vec_sr((__v2du)a, rshift); + result = (__v2du)vec_sel((__v2du)shmask, (__v2du)result, (__v2du)shmask); + + return (__m128i)result; +} + +Y_FORCE_INLINE void _mm_storeu_si128(__m128i* p, __m128i a) { + vec_vsx_st(a, 0, p); +} + +Y_FORCE_INLINE void _mm_store_si128(__m128i* p, __m128i a) { + vec_st((__v16qu)a, 0, (__v16qu*)p); +} + +Y_FORCE_INLINE __m128i _mm_unpackhi_epi8(__m128i a, __m128i b) { + return (__m128i)vec_mergel((__v16qu)a, (__v16qu)b); +} + +Y_FORCE_INLINE __m128i _mm_unpackhi_epi16(__m128i a, __m128i b) { + return (__m128i)vec_mergel((__v8hu)a, (__v8hu)b); +} + +Y_FORCE_INLINE __m128i _mm_unpackhi_epi32(__m128i a, __m128i b) { + return (__m128i)vec_mergel((__v4su)a, (__v4su)b); +} + +Y_FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) { + return (__m128i)vec_mergel((vector long long)a, (vector long long)b); +} + +Y_FORCE_INLINE __m128i _mm_unpacklo_epi8(__m128i a, __m128i b) { + return (__m128i)vec_mergeh((__v16qu)a, (__v16qu)b); +} + +Y_FORCE_INLINE __m128i _mm_unpacklo_epi16(__m128i a, __m128i b) { + return (__m128i)vec_mergeh((__v8hi)a, (__v8hi)b); +} + +Y_FORCE_INLINE __m128i _mm_unpacklo_epi32(__m128i a, __m128i b) { + return (__m128i)vec_mergeh((__v4si)a, (__v4si)b); +} + +Y_FORCE_INLINE __m128i _mm_unpacklo_epi64(__m128i a, __m128i b) { + return (__m128i)vec_mergeh((vector long long)a, (vector long long)b); +} - return (__m128i)result; +Y_FORCE_INLINE __m128i _mm_add_epi8(__m128i a, __m128i b) { + return (__m128i)((__v16qu)a + (__v16qu)b); } -Y_FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) { - __v2du lshift, shmask; - const __v2du shmax = {64, 64}; - __v2du result; +Y_FORCE_INLINE __m128i _mm_add_epi16(__m128i a, __m128i b) { + return (__m128i)((__v8hu)a + (__v8hu)b); +} + +Y_FORCE_INLINE __m128i _mm_add_epi32(__m128i a, __m128i b) { + return (__m128i)((__v4su)a + (__v4su)b); +} + +Y_FORCE_INLINE __m128i _mm_add_epi64(__m128i a, __m128i b) { + return (__m128i)((__v2du)a + (__v2du)b); +} + +Y_FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) { + const vector signed int zero = {0, 0, 0, 0}; + return (__m128i)vec_vmsumshm((__v8hi)a, (__v8hi)b, zero); +} + +Y_FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) { + return (__m128i)((__v16qu)a - (__v16qu)b); +} + +Y_FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) { + return (__m128i)((__v8hu)a - (__v8hu)b); +} + +Y_FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) { + return (__m128i)((__v4su)a - (__v4su)b); +} + +Y_FORCE_INLINE __m128i _mm_sub_epi64(__m128i a, __m128i b) { + return (__m128i)((__v2du)a - (__v2du)b); +} + +Y_FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128 b) { +#ifdef __LITTLE_ENDIAN__ + return (__m128i)vec_mule((__v4su)a, (__v4su)b); +#elif __BIG_ENDIAN__ + return (__m128i)vec_mulo((__v4su)a, (__v4su)b); +#endif +} + +Y_FORCE_INLINE __m128i _mm_set_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) { + return (__m128i)(__v16qi){q00, q01, q02, q03, q04, q05, q06, q07, q08, q09, q10, q11, q12, q13, q14, q15}; +}; + +Y_FORCE_INLINE __m128i _mm_setr_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) { + return (__m128i)(__v16qi){q15, q14, q13, q12, q11, q10, q09, q08, q07, q06, q05, q04, q03, q02, q01, q00}; +}; + +Y_FORCE_INLINE __m128i _mm_set_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) { + return (__m128i)(__v8hi){q0, q1, q2, q3, q4, q5, q6, q7}; +} - lshift = (__v2du)vec_splat((__v2du)count, 0); - shmask = vec_cmplt(lshift, shmax); - result = vec_sl((__v2du)a, lshift); - result = ((vector long long)shmask & ~(vector long long)shmask) | ((vector long long)result & (vector long long)shmask); - - return (__m128i)result; +Y_FORCE_INLINE __m128i _mm_setr_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) { + return (__m128i)(__v8hi){q7, q6, q5, q4, q3, q2, q1, q0}; } -Y_FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) { - __v8hu rshift, shmask; - const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15}; - __v8hu result; - -#ifdef __LITTLE_ENDIAN__ - rshift = vec_splat((__v8hu)count, 0); -#elif __BIG_ENDIAN__ - rshift = vec_splat((__v8hu)count, 3); -#endif - shmask = vec_cmple(rshift, shmax); - result = vec_vsrh((__v8hu)a, rshift); - result = vec_sel(shmask, result, shmask); - - return (__m128i)result; +Y_FORCE_INLINE __m128i _mm_set_epi32(int q3, int q2, int q1, int q0) { + return (__m128i)(__v4si){q0, q1, q2, q3}; } -Y_FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) { - __v4su rshift, shmask; - const __v4su shmax = {32, 32, 32, 32}; - __v4su result; - -#ifdef __LITTLE_ENDIAN__ - rshift = vec_splat((__v4su)count, 0); -#elif __BIG_ENDIAN__ - rshift = vec_splat((__v4su)count, 1); -#endif - shmask = vec_cmplt(rshift, shmax); - result = vec_vsrw((__v4su)a, rshift); - result = vec_sel(shmask, result, shmask); - - return (__m128i)result; +Y_FORCE_INLINE __m128i _mm_setr_epi32(int q3, int q2, int q1, int q0) { + return (__m128i)(__v4si){q3, q2, q1, q0}; } -Y_FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) { - __v2du rshift, shmask; - const __v2du shmax = {64, 64}; - __v2du result; - - rshift = (__v2du)vec_splat((__v2du)count, 0); - shmask = vec_cmplt(rshift, shmax); - result = vec_sr((__v2du)a, rshift); - result = (__v2du)vec_sel((__v2du)shmask, (__v2du)result, (__v2du)shmask); - - return (__m128i)result; +Y_FORCE_INLINE __m128i _mm_set1_epi8(char a) { + return _mm_set_epi8(a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a); } -Y_FORCE_INLINE void _mm_storeu_si128(__m128i* p, __m128i a) { - vec_vsx_st(a, 0, p); +Y_FORCE_INLINE __m128i _mm_set1_epi16(short a) { + return _mm_set_epi16(a, a, a, a, a, a, a, a); } -Y_FORCE_INLINE void _mm_store_si128(__m128i* p, __m128i a) { - vec_st((__v16qu)a, 0, (__v16qu*)p); +Y_FORCE_INLINE __m128i _mm_set1_epi32(int a) { + return _mm_set_epi32(a, a, a, a); } -Y_FORCE_INLINE __m128i _mm_unpackhi_epi8(__m128i a, __m128i b) { - return (__m128i)vec_mergel((__v16qu)a, (__v16qu)b); +Y_FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) { + return (__m128i)vec_cmpeq((__v16qi)a, (__v16qi)b); } -Y_FORCE_INLINE __m128i _mm_unpackhi_epi16(__m128i a, __m128i b) { - return (__m128i)vec_mergel((__v8hu)a, (__v8hu)b); +Y_FORCE_INLINE __m128i _mm_cmpeq_epi16(__m128i a, __m128i b) { + return (__m128i)vec_cmpeq((__v8hi)a, (__v8hi)b); } -Y_FORCE_INLINE __m128i _mm_unpackhi_epi32(__m128i a, __m128i b) { - return (__m128i)vec_mergel((__v4su)a, (__v4su)b); +Y_FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i a, __m128i b) { + return (__m128i)vec_cmpeq((__v4si)a, (__v4si)b); } -Y_FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) { - return (__m128i)vec_mergel((vector long long)a, (vector long long)b); +Y_FORCE_INLINE __m128i _mm_packs_epi16(__m128i a, __m128i b) { + return (__m128i)vec_packs((__v8hi)a, (__v8hi)b); } -Y_FORCE_INLINE __m128i _mm_unpacklo_epi8(__m128i a, __m128i b) { - return (__m128i)vec_mergeh((__v16qu)a, (__v16qu)b); +Y_FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) { + return (__m128i)vec_packs((__v4si)a, (__v4si)b); } -Y_FORCE_INLINE __m128i _mm_unpacklo_epi16(__m128i a, __m128i b) { - return (__m128i)vec_mergeh((__v8hi)a, (__v8hi)b); +Y_FORCE_INLINE __m128i _mm_packus_epi16(__m128i a, __m128i b) { + return (__m128i)vec_packsu((vector signed short)a, (vector signed short)b); } -Y_FORCE_INLINE __m128i _mm_unpacklo_epi32(__m128i a, __m128i b) { - return (__m128i)vec_mergeh((__v4si)a, (__v4si)b); +Y_FORCE_INLINE __m128i _mm_cvtsi64_si128(i64 a) { + return (__m128i)(__v2di){a, 0LL}; } -Y_FORCE_INLINE __m128i _mm_unpacklo_epi64(__m128i a, __m128i b) { - return (__m128i)vec_mergeh((vector long long)a, (vector long long)b); +Y_FORCE_INLINE __m128i _mm_cvtsi32_si128(int a) { + return _mm_set_epi32(0, 0, 0, a); } -Y_FORCE_INLINE __m128i _mm_add_epi8(__m128i a, __m128i b) { - return (__m128i)((__v16qu)a + (__v16qu)b); -} - -Y_FORCE_INLINE __m128i _mm_add_epi16(__m128i a, __m128i b) { - return (__m128i)((__v8hu)a + (__v8hu)b); +Y_FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) { + return ((__v4si)a)[0]; } -Y_FORCE_INLINE __m128i _mm_add_epi32(__m128i a, __m128i b) { - return (__m128i)((__v4su)a + (__v4su)b); +Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) { + return ((__v2di)a)[0]; } -Y_FORCE_INLINE __m128i _mm_add_epi64(__m128i a, __m128i b) { - return (__m128i)((__v2du)a + (__v2du)b); +Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* p) { + return *p; } -Y_FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) { - const vector signed int zero = {0, 0, 0, 0}; - return (__m128i)vec_vmsumshm((__v8hi)a, (__v8hi)b, zero); +Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* p) { + return (__m128i)(vec_vsx_ld(0, (signed int const*)p)); } -Y_FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) { - return (__m128i)((__v16qu)a - (__v16qu)b); -} - -Y_FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) { - return (__m128i)((__v8hu)a - (__v8hu)b); +Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* p) { + return _mm_loadu_si128(p); } -Y_FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) { - return (__m128i)((__v4su)a - (__v4su)b); +Y_FORCE_INLINE __m128i _mm_loadl_epi64(const __m128i* a) { +#ifdef __LITTLE_ENDIAN__ + const vector bool long long mask = { + 0xFFFFFFFFFFFFFFFFull, 0x0000000000000000ull}; +#elif __BIG_ENDIAN__ + const vector bool long long mask = { + 0x0000000000000000ull, 0xFFFFFFFFFFFFFFFFull}; +#endif + return (__m128i)vec_and(_mm_loadu_si128(a), (vector unsigned char)mask); +} + +Y_FORCE_INLINE void _mm_storel_epi64(__m128i* a, __m128i b) { + *(long long*)a = ((__v2di)b)[0]; +} + +Y_FORCE_INLINE double _mm_cvtsd_f64(__m128d a) { + return ((__v2df)a)[0]; +} + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +Y_FORCE_INLINE __m128d _mm_undefined_pd(void) { + __m128d ans = ans; + return ans; +} +#pragma GCC diagnostic pop + +Y_FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double* b) { + __v2df result = (__v2df)a; + result[1] = *b; + return (__m128d)result; +} + +Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) { + __v2df result = (__v2df)a; + result[0] = *b; + return (__m128d)result; +} + +Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i a) { + return (__m128)a; +} + +Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 a) { + return (__m128i)a; +} + +Y_FORCE_INLINE __m128i _mm_cmpgt_epi8(__m128i a, __m128i b) { + return (__m128i)vec_cmpgt((__v16qi)a, (__v16qi)b); +} + +Y_FORCE_INLINE __m128i _mm_cmpgt_epi16(__m128i a, __m128i b) { + return (__m128i)vec_cmpgt((__v8hi)a, (__v8hi)b); +} + +Y_FORCE_INLINE __m128i _mm_cmpgt_epi32(__m128i a, __m128i b) { + return (__m128i)vec_cmpgt((__v4si)a, (__v4si)b); } -Y_FORCE_INLINE __m128i _mm_sub_epi64(__m128i a, __m128i b) { - return (__m128i)((__v2du)a - (__v2du)b); +Y_FORCE_INLINE __m128i _mm_cmpgt_epi64(__m128i a, __m128i b) { + return vec_cmpgt((vector signed long long)a, (vector signed long long)b); } -Y_FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128 b) { -#ifdef __LITTLE_ENDIAN__ - return (__m128i)vec_mule((__v4su)a, (__v4su)b); -#elif __BIG_ENDIAN__ - return (__m128i)vec_mulo((__v4su)a, (__v4su)b); -#endif +Y_FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) { + return (__m128i)vec_cmplt((__v16qi)a, (__v16qi)b); } -Y_FORCE_INLINE __m128i _mm_set_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) { - return (__m128i)(__v16qi){q00, q01, q02, q03, q04, q05, q06, q07, q08, q09, q10, q11, q12, q13, q14, q15}; -}; - -Y_FORCE_INLINE __m128i _mm_setr_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) { - return (__m128i)(__v16qi){q15, q14, q13, q12, q11, q10, q09, q08, q07, q06, q05, q04, q03, q02, q01, q00}; -}; - -Y_FORCE_INLINE __m128i _mm_set_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) { - return (__m128i)(__v8hi){q0, q1, q2, q3, q4, q5, q6, q7}; +Y_FORCE_INLINE __m128i _mm_cmplt_epi16(__m128i a, __m128i b) { + return (__m128i)vec_cmplt((__v8hi)a, (__v8hi)b); } -Y_FORCE_INLINE __m128i _mm_setr_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) { - return (__m128i)(__v8hi){q7, q6, q5, q4, q3, q2, q1, q0}; -} - -Y_FORCE_INLINE __m128i _mm_set_epi32(int q3, int q2, int q1, int q0) { - return (__m128i)(__v4si){q0, q1, q2, q3}; +Y_FORCE_INLINE __m128i _mm_cmplt_epi32(__m128i a, __m128i b) { + return (__m128i)vec_cmplt((__v4si)a, (__v4si)b); } -Y_FORCE_INLINE __m128i _mm_setr_epi32(int q3, int q2, int q1, int q0) { - return (__m128i)(__v4si){q3, q2, q1, q0}; -} - -Y_FORCE_INLINE __m128i _mm_set1_epi8(char a) { - return _mm_set_epi8(a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a); -} - -Y_FORCE_INLINE __m128i _mm_set1_epi16(short a) { - return _mm_set_epi16(a, a, a, a, a, a, a, a); -} - -Y_FORCE_INLINE __m128i _mm_set1_epi32(int a) { - return _mm_set_epi32(a, a, a, a); -} - -Y_FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) { - return (__m128i)vec_cmpeq((__v16qi)a, (__v16qi)b); +Y_FORCE_INLINE __m128i _mm_cmplt_epi64(__m128i a, __m128i b) { + return vec_cmplt((vector signed long long)a, (vector signed long long)b); } -Y_FORCE_INLINE __m128i _mm_cmpeq_epi16(__m128i a, __m128i b) { - return (__m128i)vec_cmpeq((__v8hi)a, (__v8hi)b); +Y_FORCE_INLINE __m128i _mm_sad_epu8(__m128i A, __m128i B) { + __v16qu a, b; + __v16qu vmin, vmax, vabsdiff; + __v4si vsum; + const __v4su zero = {0, 0, 0, 0}; + __v4si result; + + a = (__v16qu)A; + b = (__v16qu)B; + vmin = vec_min(a, b); + vmax = vec_max(a, b); + vabsdiff = vec_sub(vmax, vmin); + /* Sum four groups of bytes into integers. */ + vsum = (__vector signed int)vec_sum4s(vabsdiff, zero); + /* Sum across four integers with two integer results. */ + result = vec_sum2s(vsum, (__vector signed int)zero); + /* Rotate the sums into the correct position. */ +#ifdef __LITTLE_ENDIAN__ + result = vec_sld(result, result, 4); +#elif __BIG_ENDIAN__ + result = vec_sld(result, result, 6); +#endif + /* Rotate the sums into the correct position. */ + return (__m128i)result; } -Y_FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i a, __m128i b) { - return (__m128i)vec_cmpeq((__v4si)a, (__v4si)b); +Y_FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) { + return (__m128i)vec_subs((__v16qi)a, (__v16qi)b); } -Y_FORCE_INLINE __m128i _mm_packs_epi16(__m128i a, __m128i b) { - return (__m128i)vec_packs((__v8hi)a, (__v8hi)b); +Y_FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) { + return (__m128i)vec_subs((__v8hi)a, (__v8hi)b); } -Y_FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) { - return (__m128i)vec_packs((__v4si)a, (__v4si)b); +Y_FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) { + return (__m128i)vec_subs((__v16qu)a, (__v16qu)b); } -Y_FORCE_INLINE __m128i _mm_packus_epi16(__m128i a, __m128i b) { - return (__m128i)vec_packsu((vector signed short)a, (vector signed short)b); +Y_FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) { + return (__m128i)vec_subs((__v8hu)a, (__v8hu)b); } -Y_FORCE_INLINE __m128i _mm_cvtsi64_si128(i64 a) { - return (__m128i)(__v2di){a, 0LL}; +Y_FORCE_INLINE __m128i _mm_adds_epi8(__m128i a, __m128i b) { + return (__m128i)vec_adds((__v16qi)a, (__v16qi)b); } -Y_FORCE_INLINE __m128i _mm_cvtsi32_si128(int a) { - return _mm_set_epi32(0, 0, 0, a); +Y_FORCE_INLINE __m128i _mm_adds_epi16(__m128i a, __m128i b) { + return (__m128i)vec_adds((__v8hi)a, (__v8hi)b); } -Y_FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) { - return ((__v4si)a)[0]; +Y_FORCE_INLINE __m128i _mm_adds_epu8(__m128i a, __m128i b) { + return (__m128i)vec_adds((__v16qu)a, (__v16qu)b); } -Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) { - return ((__v2di)a)[0]; +Y_FORCE_INLINE __m128i _mm_adds_epu16(__m128i a, __m128i b) { + return (__m128i)vec_adds((__v8hu)a, (__v8hu)b); } -Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* p) { - return *p; +Y_FORCE_INLINE __m128d _mm_castsi128_pd(__m128i a) { + return (__m128d)a; } -Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* p) { - return (__m128i)(vec_vsx_ld(0, (signed int const*)p)); +Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) { + __builtin_prefetch(p); } -Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* p) { - return _mm_loadu_si128(p); -} - -Y_FORCE_INLINE __m128i _mm_loadl_epi64(const __m128i* a) { -#ifdef __LITTLE_ENDIAN__ - const vector bool long long mask = { - 0xFFFFFFFFFFFFFFFFull, 0x0000000000000000ull}; -#elif __BIG_ENDIAN__ - const vector bool long long mask = { - 0x0000000000000000ull, 0xFFFFFFFFFFFFFFFFull}; -#endif - return (__m128i)vec_and(_mm_loadu_si128(a), (vector unsigned char)mask); -} - -Y_FORCE_INLINE void _mm_storel_epi64(__m128i* a, __m128i b) { - *(long long*)a = ((__v2di)b)[0]; -} - -Y_FORCE_INLINE double _mm_cvtsd_f64(__m128d a) { - return ((__v2df)a)[0]; -} - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wuninitialized" -Y_FORCE_INLINE __m128d _mm_undefined_pd(void) { - __m128d ans = ans; - return ans; -} -#pragma GCC diagnostic pop - -Y_FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double* b) { - __v2df result = (__v2df)a; - result[1] = *b; - return (__m128d)result; -} - -Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) { - __v2df result = (__v2df)a; - result[0] = *b; - return (__m128d)result; -} - -Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i a) { - return (__m128)a; -} - -Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 a) { - return (__m128i)a; -} - -Y_FORCE_INLINE __m128i _mm_cmpgt_epi8(__m128i a, __m128i b) { - return (__m128i)vec_cmpgt((__v16qi)a, (__v16qi)b); -} - -Y_FORCE_INLINE __m128i _mm_cmpgt_epi16(__m128i a, __m128i b) { - return (__m128i)vec_cmpgt((__v8hi)a, (__v8hi)b); -} - -Y_FORCE_INLINE __m128i _mm_cmpgt_epi32(__m128i a, __m128i b) { - return (__m128i)vec_cmpgt((__v4si)a, (__v4si)b); -} - -Y_FORCE_INLINE __m128i _mm_cmpgt_epi64(__m128i a, __m128i b) { - return vec_cmpgt((vector signed long long)a, (vector signed long long)b); -} - -Y_FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) { - return (__m128i)vec_cmplt((__v16qi)a, (__v16qi)b); -} - -Y_FORCE_INLINE __m128i _mm_cmplt_epi16(__m128i a, __m128i b) { - return (__m128i)vec_cmplt((__v8hi)a, (__v8hi)b); -} - -Y_FORCE_INLINE __m128i _mm_cmplt_epi32(__m128i a, __m128i b) { - return (__m128i)vec_cmplt((__v4si)a, (__v4si)b); -} - -Y_FORCE_INLINE __m128i _mm_cmplt_epi64(__m128i a, __m128i b) { - return vec_cmplt((vector signed long long)a, (vector signed long long)b); -} - -Y_FORCE_INLINE __m128i _mm_sad_epu8(__m128i A, __m128i B) { - __v16qu a, b; - __v16qu vmin, vmax, vabsdiff; - __v4si vsum; - const __v4su zero = {0, 0, 0, 0}; - __v4si result; - - a = (__v16qu)A; - b = (__v16qu)B; - vmin = vec_min(a, b); - vmax = vec_max(a, b); - vabsdiff = vec_sub(vmax, vmin); - /* Sum four groups of bytes into integers. */ - vsum = (__vector signed int)vec_sum4s(vabsdiff, zero); - /* Sum across four integers with two integer results. */ - result = vec_sum2s(vsum, (__vector signed int)zero); - /* Rotate the sums into the correct position. */ -#ifdef __LITTLE_ENDIAN__ - result = vec_sld(result, result, 4); -#elif __BIG_ENDIAN__ - result = vec_sld(result, result, 6); -#endif - /* Rotate the sums into the correct position. */ - return (__m128i)result; -} - -Y_FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) { - return (__m128i)vec_subs((__v16qi)a, (__v16qi)b); -} - -Y_FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) { - return (__m128i)vec_subs((__v8hi)a, (__v8hi)b); -} - -Y_FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) { - return (__m128i)vec_subs((__v16qu)a, (__v16qu)b); -} - -Y_FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) { - return (__m128i)vec_subs((__v8hu)a, (__v8hu)b); -} - -Y_FORCE_INLINE __m128i _mm_adds_epi8(__m128i a, __m128i b) { - return (__m128i)vec_adds((__v16qi)a, (__v16qi)b); -} - -Y_FORCE_INLINE __m128i _mm_adds_epi16(__m128i a, __m128i b) { - return (__m128i)vec_adds((__v8hi)a, (__v8hi)b); -} - -Y_FORCE_INLINE __m128i _mm_adds_epu8(__m128i a, __m128i b) { - return (__m128i)vec_adds((__v16qu)a, (__v16qu)b); -} - -Y_FORCE_INLINE __m128i _mm_adds_epu16(__m128i a, __m128i b) { - return (__m128i)vec_adds((__v8hu)a, (__v8hu)b); -} - -Y_FORCE_INLINE __m128d _mm_castsi128_pd(__m128i a) { - return (__m128d)a; -} - -Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) { - __builtin_prefetch(p); -} - -Y_FORCE_INLINE __m128i _mm_hadd_epi16(__m128i a, __m128i b) { - const __v16qu p = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 }; - const __v16qu q = { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 }; - __v8hi c = vec_perm((__v8hi)a, (__v8hi)b, p); - __v8hi d = vec_perm((__v8hi)a, (__v8hi)b, q); - return (__m128i)vec_add(c, d); -} +Y_FORCE_INLINE __m128i _mm_hadd_epi16(__m128i a, __m128i b) { + const __v16qu p = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 }; + const __v16qu q = { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 }; + __v8hi c = vec_perm((__v8hi)a, (__v8hi)b, p); + __v8hi d = vec_perm((__v8hi)a, (__v8hi)b, q); + return (__m128i)vec_add(c, d); +} diff --git a/library/cpp/sse/sse.cpp b/library/cpp/sse/sse.cpp index 1e880e40d3..d0d1de9930 100644 --- a/library/cpp/sse/sse.cpp +++ b/library/cpp/sse/sse.cpp @@ -1 +1 @@ -#include "sse.h" +#include "sse.h" diff --git a/library/cpp/sse/sse.h b/library/cpp/sse/sse.h index 39d1ccb958..19bac17de0 100644 --- a/library/cpp/sse/sse.h +++ b/library/cpp/sse/sse.h @@ -4,16 +4,16 @@ The header chooses appropriate SSE support. On Intel: SSE intrinsics On ARM64: translation to NEON intrinsics or software emulation - On PowerPc: translation to Altivec intrinsics or software emulation + On PowerPc: translation to Altivec intrinsics or software emulation */ -/* Author: Vitaliy Manushkin <agri@yandex-team.ru>, Danila Kutenin <danlark@yandex-team.ru> */ +/* Author: Vitaliy Manushkin <agri@yandex-team.ru>, Danila Kutenin <danlark@yandex-team.ru> */ #include <util/system/platform.h> #if (defined(_i386_) || defined(_x86_64_)) && defined(_sse_) #include <xmmintrin.h> #include <emmintrin.h> -#include <pmmintrin.h> +#include <pmmintrin.h> #define ARCADIA_SSE #if defined(_ssse3_) #include <tmmintrin.h> diff --git a/library/cpp/sse/sse2neon.h b/library/cpp/sse/sse2neon.h index f60711e77f..695dbd3041 100644 --- a/library/cpp/sse/sse2neon.h +++ b/library/cpp/sse/sse2neon.h @@ -17,7 +17,7 @@ #if !defined(_arm64_) #error "This header is for ARM64 (aarch64) platform only. " \ - "Include sse.h instead of including this header directly." + "Include sse.h instead of including this header directly." #endif #include <arm_neon.h> @@ -57,21 +57,21 @@ union __m128 { typedef float64x2_t __m128d; -enum _mm_hint -{ - /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ - _MM_HINT_ET0 = 7, - _MM_HINT_ET1 = 6, - _MM_HINT_T0 = 3, - _MM_HINT_T1 = 2, - _MM_HINT_T2 = 1, - _MM_HINT_NTA = 0 -}; - -Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) { - __builtin_prefetch(p); -} - +enum _mm_hint +{ + /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ + _MM_HINT_ET0 = 7, + _MM_HINT_ET1 = 6, + _MM_HINT_T0 = 3, + _MM_HINT_T1 = 2, + _MM_HINT_T2 = 1, + _MM_HINT_NTA = 0 +}; + +Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) { + __builtin_prefetch(p); +} + template <typename TType> struct TQType; @@ -299,9 +299,9 @@ using _mm_or_si128 = TWrapperDual<uint64x2_t, decltype(vorrq_u64), vorrq_u64>; using _mm_and_si128 = TWrapperDual<uint64x2_t, decltype(vandq_u64), vandq_u64>; using _mm_andnot_si128 = TWrapperDualSwap<uint64x2_t, decltype(vbicq_u64), vbicq_u64>; -using _mm_xor_si128 = TWrapperDual<uint64x2_t, decltype(veorq_u64), veorq_u64>; +using _mm_xor_si128 = TWrapperDual<uint64x2_t, decltype(veorq_u64), veorq_u64>; -using _mm_add_epi8 = TWrapperDual<uint8x16_t, decltype(vaddq_u8), vaddq_u8>; +using _mm_add_epi8 = TWrapperDual<uint8x16_t, decltype(vaddq_u8), vaddq_u8>; using _mm_add_epi16 = TWrapperDual<uint16x8_t, decltype(vaddq_u16), vaddq_u16>; using _mm_add_epi32 = TWrapperDual<uint32x4_t, decltype(vaddq_u32), vaddq_u32>; using _mm_add_epi64 = TWrapperDual<uint64x2_t, decltype(vaddq_u64), vaddq_u64>; @@ -342,7 +342,7 @@ inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { return res; } -using _mm_sub_epi8 = TWrapperDual<uint8x16_t, decltype(vsubq_u8), vsubq_u8>; +using _mm_sub_epi8 = TWrapperDual<uint8x16_t, decltype(vsubq_u8), vsubq_u8>; using _mm_sub_epi16 = TWrapperDual<uint16x8_t, decltype(vsubq_u16), vsubq_u16>; using _mm_sub_epi32 = TWrapperDual<uint32x4_t, decltype(vsubq_u32), vsubq_u32>; using _mm_sub_epi64 = TWrapperDual<uint64x2_t, decltype(vsubq_u64), vsubq_u64>; @@ -385,27 +385,27 @@ using _mm_cmplt_epi16 = using _mm_cmplt_epi32 = TWrapperDual<int32x4_t, decltype(vcltq_s32), vcltq_s32>; -Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* ptr) { +Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* ptr) { __m128i result; result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); return result; } -Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* ptr) { +Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* ptr) { __m128i result; result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); return result; } -Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* ptr) { - return _mm_loadu_si128(ptr); -} - -Y_FORCE_INLINE void _mm_storeu_si128(__m128i* ptr, const __m128i& op) { +Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* ptr) { + return _mm_loadu_si128(ptr); +} + +Y_FORCE_INLINE void _mm_storeu_si128(__m128i* ptr, const __m128i& op) { vst1q_u64((uint64_t*)ptr, op.AsUi64x2); } -Y_FORCE_INLINE void +Y_FORCE_INLINE void _mm_store_si128(__m128i* ptr, const __m128i& op) { vst1q_u64((uint64_t*)ptr, op.AsUi64x2); } @@ -459,7 +459,7 @@ struct ShuffleStruct4 { ui8 x[4]; }; -Y_FORCE_INLINE ShuffleStruct4 +Y_FORCE_INLINE ShuffleStruct4 _MM_SHUFFLE(ui8 x4, ui8 x3, ui8 x2, ui8 x1) { ShuffleStruct4 result; result.x[0] = x1; @@ -469,7 +469,7 @@ _MM_SHUFFLE(ui8 x4, ui8 x3, ui8 x2, ui8 x1) { return result; } -Y_FORCE_INLINE __m128i +Y_FORCE_INLINE __m128i _mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) { __m128i result; const ui8 xi[4] = { @@ -486,7 +486,7 @@ _mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) { return result; } -Y_FORCE_INLINE int +Y_FORCE_INLINE int _mm_movemask_epi8(const __m128i& op) { uint8x16_t mask = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; @@ -542,7 +542,7 @@ struct THelper_mm_slli_si128 : TBaseWrapper<__m128i> { #define _mm_slli_si128(a, imm) THelper_mm_slli_si128<imm>(a) -Y_FORCE_INLINE int _mm_cvtsi128_si32(const __m128i& op) { +Y_FORCE_INLINE int _mm_cvtsi128_si32(const __m128i& op) { return vgetq_lane_s32(op.AsSi32x4, 0); } @@ -558,18 +558,18 @@ struct _mm_set_epi16 : TBaseWrapper<__m128i> { } }; -struct _mm_setr_epi16 : TBaseWrapper<__m128i> { - Y_FORCE_INLINE - _mm_setr_epi16(const short w7, const short w6, - const short w5, const short w4, - const short w3, const short w2, - const short w1, const short w0) { - int16x4_t d0 = {w7, w6, w5, w4}; - int16x4_t d1 = {w3, w2, w1, w0}; - TQType<int16x8_t>::As(Value) = vcombine_s16(d0, d1); - } -}; - +struct _mm_setr_epi16 : TBaseWrapper<__m128i> { + Y_FORCE_INLINE + _mm_setr_epi16(const short w7, const short w6, + const short w5, const short w4, + const short w3, const short w2, + const short w1, const short w0) { + int16x4_t d0 = {w7, w6, w5, w4}; + int16x4_t d1 = {w3, w2, w1, w0}; + TQType<int16x8_t>::As(Value) = vcombine_s16(d0, d1); + } +}; + struct _mm_set_epi32 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_set_epi32(const int x3, const int x2, @@ -580,16 +580,16 @@ struct _mm_set_epi32 : TBaseWrapper<__m128i> { } }; -struct _mm_setr_epi32 : TBaseWrapper<__m128i> { - Y_FORCE_INLINE - _mm_setr_epi32(const int x3, const int x2, - const int x1, const int x0) { - int32x2_t d0 = {x3, x2}; - int32x2_t d1 = {x1, x0}; - TQType<int32x4_t>::As(Value) = vcombine_s32(d0, d1); - } -}; - +struct _mm_setr_epi32 : TBaseWrapper<__m128i> { + Y_FORCE_INLINE + _mm_setr_epi32(const int x3, const int x2, + const int x1, const int x0) { + int32x2_t d0 = {x3, x2}; + int32x2_t d1 = {x1, x0}; + TQType<int32x4_t>::As(Value) = vcombine_s32(d0, d1); + } +}; + struct _mm_cvtsi32_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_cvtsi32_si128(int op) { @@ -642,31 +642,31 @@ struct TScalarOutWrapper : TBaseWrapper<TOpOut> { }; template<int imm> -int extract_epi8_arm(__m128i arg) { - return vgetq_lane_u8(arg.AsUi8x16, imm); -} - -template<int imm> -int extract_epi16_arm(__m128i arg) { +int extract_epi8_arm(__m128i arg) { + return vgetq_lane_u8(arg.AsUi8x16, imm); +} + +template<int imm> +int extract_epi16_arm(__m128i arg) { return vgetq_lane_u16(arg.AsUi16x8, imm); } -template<int imm> -int extract_epi32_arm(__m128i arg) { - return vgetq_lane_s32(arg.AsSi32x4, imm); -} - -template<int imm> -long long extract_epi64_arm(__m128i arg) { - return vgetq_lane_s64(arg.AsSi64x2, imm); -} - -#define _mm_extract_epi8(op, imm) extract_epi8_arm<imm>(op) -#define _mm_extract_epi16(op, imm) extract_epi16_arm<imm>(op) -#define _mm_extract_epi32(op, imm) extract_epi32_arm<imm>(op) -#define _mm_extract_epi64(op, imm) extract_epi64_arm<imm>(op) -#define _mm_extract_ps(op, imm) _mm_extract_epi32(op, imm) - +template<int imm> +int extract_epi32_arm(__m128i arg) { + return vgetq_lane_s32(arg.AsSi32x4, imm); +} + +template<int imm> +long long extract_epi64_arm(__m128i arg) { + return vgetq_lane_s64(arg.AsSi64x2, imm); +} + +#define _mm_extract_epi8(op, imm) extract_epi8_arm<imm>(op) +#define _mm_extract_epi16(op, imm) extract_epi16_arm<imm>(op) +#define _mm_extract_epi32(op, imm) extract_epi32_arm<imm>(op) +#define _mm_extract_epi64(op, imm) extract_epi64_arm<imm>(op) +#define _mm_extract_ps(op, imm) _mm_extract_epi32(op, imm) + static Y_FORCE_INLINE __m128i _mm_mul_epu32(__m128i op1, __m128i op2) { __m128i result; @@ -734,27 +734,27 @@ struct _mm_setzero_ps : TBaseWrapper<__m128> { } }; -Y_FORCE_INLINE __m128d _mm_setzero_pd() { +Y_FORCE_INLINE __m128d _mm_setzero_pd() { return vdupq_n_f64(0.); } -Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* ptr) { +Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* ptr) { __m128 result; result.AsFloat32x4 = vld1q_f32(ptr); return result; } -Y_FORCE_INLINE __m128 _mm_load_ps(const float* ptr) { +Y_FORCE_INLINE __m128 _mm_load_ps(const float* ptr) { __m128 result; result.AsFloat32x4 = vld1q_f32(ptr); return result; } -Y_FORCE_INLINE void _mm_storeu_ps(float* ptr, const __m128& op) { +Y_FORCE_INLINE void _mm_storeu_ps(float* ptr, const __m128& op) { vst1q_f32(ptr, op.AsFloat32x4); } -Y_FORCE_INLINE void _mm_store_ps(float* ptr, const __m128& op) { +Y_FORCE_INLINE void _mm_store_ps(float* ptr, const __m128& op) { vst1q_f32(ptr, op.AsFloat32x4); } @@ -768,23 +768,23 @@ struct _mm_set_ps : TBaseWrapper<__m128> { } }; -Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { +Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) { const float64x1_t p0 = {d0}; const float64x1_t p1 = {d1}; return vcombine_f64(p0, p1); } -Y_FORCE_INLINE __m128d _mm_loadu_pd(const double* d) { +Y_FORCE_INLINE __m128d _mm_loadu_pd(const double* d) { __m128d res; res = vld1q_f64(d); return res; } -Y_FORCE_INLINE void _mm_storeu_pd(double* res, __m128d a) { +Y_FORCE_INLINE void _mm_storeu_pd(double* res, __m128d a) { vst1q_f64(res, a); } -Y_FORCE_INLINE void _mm_store_pd(double* res, __m128d a) { +Y_FORCE_INLINE void _mm_store_pd(double* res, __m128d a) { vst1q_f64(res, a); } @@ -811,11 +811,11 @@ struct _mm_and_ps : TBaseWrapper<__m128> { } }; -Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { - return vandq_u64(a, b); -} - -Y_FORCE_INLINE void _MM_TRANSPOSE4_PS(__m128& op0, __m128& op1, __m128& op2, __m128& op3) { +Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) { + return vandq_u64(a, b); +} + +Y_FORCE_INLINE void _MM_TRANSPOSE4_PS(__m128& op0, __m128& op1, __m128& op2, __m128& op3) { float64x2_t im0 = (float64x2_t)vtrn1q_f32(op0.AsFloat32x4, op1.AsFloat32x4); float64x2_t im1 = @@ -831,11 +831,11 @@ Y_FORCE_INLINE void _MM_TRANSPOSE4_PS(__m128& op0, __m128& op1, __m128& op2, __m TQType<float64x2_t>::As(op3) = vtrn2q_f64(im1, im3); }; -Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i op) { +Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i op) { return reinterpret_cast<__m128&>(op); } -Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 op) { +Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 op) { return reinterpret_cast<__m128i&>(op); } @@ -878,7 +878,7 @@ using _mm_cvttps_epi32 = TCvtF2SWrapperSingle<int32x4_t, float32x4_t, decltype(vcvtq_s32_f32), vcvtq_s32_f32>; -Y_FORCE_INLINE int +Y_FORCE_INLINE int _mm_movemask_ps(const __m128& op) { uint32x4_t mask = {0x80000000, 0x80000000, 0x80000000, 0x80000000}; uint32x4_t bits = vandq_u32(op.AsUi32x4, mask); @@ -924,27 +924,27 @@ inline void _mm_store_ss(float* p, __m128 a) { *p = vgetq_lane_f32(a.AsFloat32x4, 0); } -inline float vgetg_lane_f32_switch(float32x4_t a, ui8 b) { - switch (b & 0x3) { - case 0: - return vgetq_lane_f32(a, 0); - case 1: - return vgetq_lane_f32(a, 1); - case 2: - return vgetq_lane_f32(a, 2); - case 3: - return vgetq_lane_f32(a, 3); - } - return 0; -} - +inline float vgetg_lane_f32_switch(float32x4_t a, ui8 b) { + switch (b & 0x3) { + case 0: + return vgetq_lane_f32(a, 0); + case 1: + return vgetq_lane_f32(a, 1); + case 2: + return vgetq_lane_f32(a, 2); + case 3: + return vgetq_lane_f32(a, 3); + } + return 0; +} + inline __m128 _mm_shuffle_ps(__m128 a, __m128 b, const ShuffleStruct4& shuf) { - __m128 ret; - ret.AsFloat32x4 = vmovq_n_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[0])); - ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[1]), ret.AsFloat32x4, 1); - ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[2]), ret.AsFloat32x4, 2); - ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[3]), ret.AsFloat32x4, 3); - return ret; + __m128 ret; + ret.AsFloat32x4 = vmovq_n_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[0])); + ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[1]), ret.AsFloat32x4, 1); + ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[2]), ret.AsFloat32x4, 2); + ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[3]), ret.AsFloat32x4, 3); + return ret; } inline __m128 _mm_or_ps(__m128 a, __m128 b) { @@ -952,94 +952,94 @@ inline __m128 _mm_or_ps(__m128 a, __m128 b) { res.AsUi32x4 = vorrq_u32(a.AsUi32x4, b.AsUi32x4); return res; } - -inline __m128i _mm_sad_epu8(__m128i a, __m128i b) { - uint16x8_t t = vpaddlq_u8(vabdq_u8(a.AsUi8x16, b.AsUi8x16)); - uint16_t r0 = t[0] + t[1] + t[2] + t[3]; - uint16_t r4 = t[4] + t[5] + t[6] + t[7]; - uint16x8_t r = vsetq_lane_u16(r0, vdupq_n_u16(0), 0); - __m128i ans; - ans.AsUi16x8 = vsetq_lane_u16(r4, r, 4); - return ans; -} - -Y_FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) { - __m128i ans; - ans.AsSi8x16 = vqsubq_s8(a.AsSi8x16, b.AsSi8x16); - return ans; -} - -Y_FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) { - __m128i ans; - ans.AsSi16x8 = vqsubq_s16(a.AsSi16x8, b.AsSi16x8); - return ans; -} - -Y_FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) { - __m128i ans; - ans.AsUi8x16 = vqsubq_u8(a.AsUi8x16, b.AsUi8x16); - return ans; -} - -Y_FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) { - __m128i ans; - ans.AsUi16x8 = vqsubq_u16(a.AsUi16x8, b.AsUi16x8); - return ans; -} - -Y_FORCE_INLINE __m128d _mm_castsi128_pd(__m128i __A) { - return reinterpret_cast<__m128d&>(__A); -} - -Y_FORCE_INLINE __m128i _mm_set_epi8(ui8 i15, ui8 i14, ui8 i13, ui8 i12, ui8 i11, ui8 i10, ui8 i9, ui8 i8, - ui8 i7, ui8 i6, ui8 i5, ui8 i4, ui8 i3, ui8 i2, ui8 i1, ui8 i0) -{ - int a0 = i0 | (i1<<8) | (i2<<16) | (i3<<24); - int a1 = i4 | (i5<<8) | (i6<<16) | (i7<<24); - int a2 = i8 | (i9<<8) | (i10<<16) | (i11<<24); - int a3 = i12 | (i13<<8) | (i14<<16) | (i15<<24); - return _mm_set_epi32(a3, a2, a1, a0); -} - -Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) { - __m128i ans; - ans.AsUi8x16 = vmaxq_u8(a.AsUi8x16, b.AsUi8x16); - return ans; -} - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wuninitialized" -Y_FORCE_INLINE __m128d _mm_undefined_pd(void) { - __m128d ans = ans; - return ans; -} -#pragma GCC diagnostic pop - -Y_FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double* b) { - a[1] = *b; - return a; -} - -Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) { - a[0] = *b; - return a; -} - -Y_FORCE_INLINE double _mm_cvtsd_f64(__m128d a) { - return a[0]; -} - -Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, int mask) { - __m128d result; - const int litmsk = mask & 0x3; - - if (litmsk == 0) - result = vzip1q_f64(a, b); - else if (litmsk == 1) - result = __builtin_shufflevector(a, b, 1, 2); - else if (litmsk == 2) - result = __builtin_shufflevector(a, b, 0, 3); - else - result = vzip2q_f64(a, b); - return result; -} + +inline __m128i _mm_sad_epu8(__m128i a, __m128i b) { + uint16x8_t t = vpaddlq_u8(vabdq_u8(a.AsUi8x16, b.AsUi8x16)); + uint16_t r0 = t[0] + t[1] + t[2] + t[3]; + uint16_t r4 = t[4] + t[5] + t[6] + t[7]; + uint16x8_t r = vsetq_lane_u16(r0, vdupq_n_u16(0), 0); + __m128i ans; + ans.AsUi16x8 = vsetq_lane_u16(r4, r, 4); + return ans; +} + +Y_FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) { + __m128i ans; + ans.AsSi8x16 = vqsubq_s8(a.AsSi8x16, b.AsSi8x16); + return ans; +} + +Y_FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) { + __m128i ans; + ans.AsSi16x8 = vqsubq_s16(a.AsSi16x8, b.AsSi16x8); + return ans; +} + +Y_FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) { + __m128i ans; + ans.AsUi8x16 = vqsubq_u8(a.AsUi8x16, b.AsUi8x16); + return ans; +} + +Y_FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) { + __m128i ans; + ans.AsUi16x8 = vqsubq_u16(a.AsUi16x8, b.AsUi16x8); + return ans; +} + +Y_FORCE_INLINE __m128d _mm_castsi128_pd(__m128i __A) { + return reinterpret_cast<__m128d&>(__A); +} + +Y_FORCE_INLINE __m128i _mm_set_epi8(ui8 i15, ui8 i14, ui8 i13, ui8 i12, ui8 i11, ui8 i10, ui8 i9, ui8 i8, + ui8 i7, ui8 i6, ui8 i5, ui8 i4, ui8 i3, ui8 i2, ui8 i1, ui8 i0) +{ + int a0 = i0 | (i1<<8) | (i2<<16) | (i3<<24); + int a1 = i4 | (i5<<8) | (i6<<16) | (i7<<24); + int a2 = i8 | (i9<<8) | (i10<<16) | (i11<<24); + int a3 = i12 | (i13<<8) | (i14<<16) | (i15<<24); + return _mm_set_epi32(a3, a2, a1, a0); +} + +Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) { + __m128i ans; + ans.AsUi8x16 = vmaxq_u8(a.AsUi8x16, b.AsUi8x16); + return ans; +} + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +Y_FORCE_INLINE __m128d _mm_undefined_pd(void) { + __m128d ans = ans; + return ans; +} +#pragma GCC diagnostic pop + +Y_FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double* b) { + a[1] = *b; + return a; +} + +Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) { + a[0] = *b; + return a; +} + +Y_FORCE_INLINE double _mm_cvtsd_f64(__m128d a) { + return a[0]; +} + +Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, int mask) { + __m128d result; + const int litmsk = mask & 0x3; + + if (litmsk == 0) + result = vzip1q_f64(a, b); + else if (litmsk == 1) + result = __builtin_shufflevector(a, b, 1, 2); + else if (litmsk == 2) + result = __builtin_shufflevector(a, b, 0, 3); + else + result = vzip2q_f64(a, b); + return result; +} diff --git a/library/cpp/sse/ut/test.cpp b/library/cpp/sse/ut/test.cpp index a4e6c2bfbc..33c999d284 100644 --- a/library/cpp/sse/ut/test.cpp +++ b/library/cpp/sse/ut/test.cpp @@ -9,15 +9,15 @@ #include <util/generic/typetraits.h> #include <util/string/hex.h> -#include <util/random/fast.h> -#include <util/stream/output.h> +#include <util/random/fast.h> +#include <util/stream/output.h> -#include <algorithm> +#include <algorithm> #include <array> #include <limits> -#include <memory> -#include <type_traits> -#include <utility> +#include <memory> +#include <type_traits> +#include <utility> template <typename TResult, typename TFunc, TFunc* func> struct T_mm_CallWrapper { @@ -42,7 +42,7 @@ struct T_mm_CallWrapper { #elif defined(_i386_) || defined(_x86_64_) #include <xmmintrin.h> #include <emmintrin.h> -#include <smmintrin.h> +#include <smmintrin.h> #elif defined(_ppc64_) #include "library/cpp/sse/powerpc.h" #else @@ -259,10 +259,10 @@ private: UNIT_TEST(Test_mm_storel_epi64); UNIT_TEST(Test_mm_loadl_epi64); - UNIT_TEST(Test_mm_loadl_pd); - UNIT_TEST(Test_mm_loadh_pd); - UNIT_TEST(Test_mm_cvtsd_f64); - + UNIT_TEST(Test_mm_loadl_pd); + UNIT_TEST(Test_mm_loadh_pd); + UNIT_TEST(Test_mm_cvtsd_f64); + UNIT_TEST(Test_mm_shuffle_epi32); UNIT_TEST(Test_mm_movemask_epi8); UNIT_TEST(Test_mm_cvtsi128_si32); @@ -281,9 +281,9 @@ private: UNIT_TEST(Test_mm_packus_epi16); UNIT_TEST(Test_mm_extract_epi16); - UNIT_TEST(Test_mm_extract_epi8); - UNIT_TEST(Test_mm_extract_epi32); - UNIT_TEST(Test_mm_extract_epi64); + UNIT_TEST(Test_mm_extract_epi8); + UNIT_TEST(Test_mm_extract_epi32); + UNIT_TEST(Test_mm_extract_epi64); UNIT_TEST(Test_MM_TRANSPOSE4_PS); UNIT_TEST(Test_mm_movemask_ps); @@ -301,14 +301,14 @@ private: UNIT_TEST(Test_mm_cmpunord_ps); UNIT_TEST(Test_mm_andnot_ps); UNIT_TEST(Test_mm_shuffle_ps); - UNIT_TEST(Test_mm_shuffle_pd); + UNIT_TEST(Test_mm_shuffle_pd); UNIT_TEST(Test_mm_or_ps); UNIT_TEST(Test_mm_store_ss); UNIT_TEST(Test_mm_store_ps); UNIT_TEST(Test_mm_storeu_pd); - UNIT_TEST(Test_mm_loadu_pd); - UNIT_TEST(Test_mm_rsqrt_ps); - UNIT_TEST(Test_matrixnet_powerpc); + UNIT_TEST(Test_mm_loadu_pd); + UNIT_TEST(Test_mm_rsqrt_ps); + UNIT_TEST(Test_matrixnet_powerpc); UNIT_TEST_SUITE_END(); @@ -436,10 +436,10 @@ public: void Test_mm_loadl_epi64(); void Test_mm_storel_epi64(); - void Test_mm_loadl_pd(); - void Test_mm_loadh_pd(); - void Test_mm_cvtsd_f64(); - + void Test_mm_loadl_pd(); + void Test_mm_loadh_pd(); + void Test_mm_cvtsd_f64(); + void Test_mm_shuffle_epi32(); void Test_mm_movemask_epi8(); void Test_mm_cvtsi128_si32(); @@ -461,9 +461,9 @@ public: void Test_mm_packus_epi16(); void Test_mm_extract_epi16(); - void Test_mm_extract_epi8(); - void Test_mm_extract_epi32(); - void Test_mm_extract_epi64(); + void Test_mm_extract_epi8(); + void Test_mm_extract_epi32(); + void Test_mm_extract_epi64(); void Test_MM_TRANSPOSE4_PS(); void Test_mm_movemask_ps(); @@ -491,12 +491,12 @@ public: void Test_mm_storeu_pd(); void Test_mm_andnot_ps(); void Test_mm_shuffle_ps(); - void Test_mm_shuffle_pd(); + void Test_mm_shuffle_pd(); void Test_mm_or_ps(); - void Test_mm_loadu_pd(); - void Test_mm_rsqrt_ps(); - void Test_mm_rsqrt_ss(); - void Test_matrixnet_powerpc(); + void Test_mm_loadu_pd(); + void Test_mm_rsqrt_ps(); + void Test_mm_rsqrt_ss(); + void Test_matrixnet_powerpc(); }; UNIT_TEST_SUITE_REGISTRATION(TSSEEmulTest); @@ -1569,33 +1569,33 @@ void TSSEEmulTest::Test_mm_packus_epi16() { Test_mm_packs_epiXX<i16, ui8, 16, Wrap(_mm_packus_epi16)>(); } -void TSSEEmulTest::Test_mm_extract_epi8() { - alignas(16) char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - const ui8* dataw = reinterpret_cast<const ui8*>(&data); - const __m128i value = _mm_loadu_si128((__m128i*)&data); - - UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 1)), int(dataw[1])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 2)), int(dataw[2])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 3)), int(dataw[3])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 4)), int(dataw[4])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 5)), int(dataw[5])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 6)), int(dataw[6])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 7)), int(dataw[7])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 8)), int(dataw[8])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 9)), int(dataw[9])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 10)), int(dataw[10])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 11)), int(dataw[11])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 12)), int(dataw[12])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 13)), int(dataw[13])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 14)), int(dataw[14])); - UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 15)), int(dataw[15])); -} - +void TSSEEmulTest::Test_mm_extract_epi8() { + alignas(16) char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + const ui8* dataw = reinterpret_cast<const ui8*>(&data); + const __m128i value = _mm_loadu_si128((__m128i*)&data); + + UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 1)), int(dataw[1])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 2)), int(dataw[2])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 3)), int(dataw[3])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 4)), int(dataw[4])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 5)), int(dataw[5])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 6)), int(dataw[6])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 7)), int(dataw[7])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 8)), int(dataw[8])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 9)), int(dataw[9])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 10)), int(dataw[10])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 11)), int(dataw[11])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 12)), int(dataw[12])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 13)), int(dataw[13])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 14)), int(dataw[14])); + UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 15)), int(dataw[15])); +} + void TSSEEmulTest::Test_mm_extract_epi16() { - alignas(16) char data[16] = { + alignas(16) char data[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; const ui16* dataw = reinterpret_cast<const ui16*>(&data); @@ -1611,30 +1611,30 @@ void TSSEEmulTest::Test_mm_extract_epi16() { UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 7)), int(dataw[7])); } -void TSSEEmulTest::Test_mm_extract_epi64() { - alignas(16) char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - const ui64* dataw = reinterpret_cast<const ui64*>(&data); - const __m128i value = _mm_loadu_si128((__m128i*)&data); - - UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 0)), (long long)(dataw[0])); - UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 1)), (long long)(dataw[1])); -} - -void TSSEEmulTest::Test_mm_extract_epi32() { - alignas(16) char data[16] = { - '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; - const ui32* dataw = reinterpret_cast<const ui32*>(&data); - const __m128i value = _mm_loadu_si128((__m128i*)&data); - - UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 0)), int(dataw[0])); - UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 1)), int(dataw[1])); - UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 2)), int(dataw[2])); - UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 3)), int(dataw[3])); -} - +void TSSEEmulTest::Test_mm_extract_epi64() { + alignas(16) char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + const ui64* dataw = reinterpret_cast<const ui64*>(&data); + const __m128i value = _mm_loadu_si128((__m128i*)&data); + + UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 0)), (long long)(dataw[0])); + UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 1)), (long long)(dataw[1])); +} + +void TSSEEmulTest::Test_mm_extract_epi32() { + alignas(16) char data[16] = { + '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'}; + const ui32* dataw = reinterpret_cast<const ui32*>(&data); + const __m128i value = _mm_loadu_si128((__m128i*)&data); + + UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 0)), int(dataw[0])); + UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 1)), int(dataw[1])); + UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 2)), int(dataw[2])); + UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 3)), int(dataw[3])); +} + void TSSEEmulTest::Test_MM_TRANSPOSE4_PS() { char data0[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', @@ -1851,17 +1851,17 @@ void TSSEEmulTest::Test_mm_store_ps() { } void TSSEEmulTest::Test_mm_storeu_pd() { - alignas(16) const double valueBits[4] = {1., 2., 3., 4.}; - for (size_t i = 0; i != 3; ++i) { - const __m128d value = _mm_loadu_pd(&valueBits[i]); - alignas(16) double res[4]; - for (size_t shift = 0; shift != 3; ++shift) { - _mm_storeu_pd(&res[shift], value); - for (size_t j = 0; j != 2; ++j) { + alignas(16) const double valueBits[4] = {1., 2., 3., 4.}; + for (size_t i = 0; i != 3; ++i) { + const __m128d value = _mm_loadu_pd(&valueBits[i]); + alignas(16) double res[4]; + for (size_t shift = 0; shift != 3; ++shift) { + _mm_storeu_pd(&res[shift], value); + for (size_t j = 0; j != 2; ++j) { UNIT_ASSERT_EQUAL_C(res[j + shift], valueBits[i + j], "res: " << HexEncode(&res[shift], 16) << " vs etalon: " << HexEncode(&valueBits[i], 16)); - } - } - } + } + } + } } void TSSEEmulTest::Test_mm_andnot_ps() { @@ -1899,75 +1899,75 @@ void TSSEEmulTest::Test_mm_shuffle_ps() { UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); } -void TSSEEmulTest::Test_mm_shuffle_pd() { - const double first[2] = {1.3, 2.3}; - const double second[2] = {5.3, 6.3}; - const double etalon0[2] = {1.3, 5.3}; - const double etalon1[2] = {2.3, 5.3}; - const double etalon2[2] = {1.3, 6.3}; - const double etalon3[2] = {2.3, 6.3}; - - const __m128d value1 = _mm_loadu_pd(first); - const __m128d value2 = _mm_loadu_pd(second); - - __m128d res = _mm_shuffle_pd(value1, value2, 0); - UNIT_ASSERT_EQUAL(::memcmp(&res, etalon0, sizeof(etalon0)), 0); - - res = _mm_shuffle_pd(value1, value2, 1); - UNIT_ASSERT_EQUAL(::memcmp(&res, etalon1, sizeof(etalon1)), 0); - - res = _mm_shuffle_pd(value1, value2, 2); - UNIT_ASSERT_EQUAL(::memcmp(&res, etalon2, sizeof(etalon2)), 0); - - res = _mm_shuffle_pd(value1, value2, 3); - UNIT_ASSERT_EQUAL(::memcmp(&res, etalon3, sizeof(etalon3)), 0); -} - -void TSSEEmulTest::Test_mm_cvtsd_f64() { - const double first[2] = {1.3, 2.3}; - const double second[2] = {5.3, 6.3}; - - const __m128d value1 = _mm_loadu_pd(first); - const __m128d value2 = _mm_loadu_pd(second); - - UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value1), 1.3); - UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value2), 5.3); -} - -void TSSEEmulTest::Test_mm_loadl_pd() { - const double first[2] = {1.3, 2.3}; - const double second[2] = {5.3, 6.3}; - const double firstEtalon[2] = {10.13, 2.3}; - const double secondEtalon[2] = {11.13, 6.3}; - - double newFirst = 10.13; - double newSecond = 11.13; - - __m128d value1 = _mm_loadu_pd(first); - __m128d value2 = _mm_loadu_pd(second); - value1 = _mm_loadl_pd(value1, &newFirst); - value2 = _mm_loadl_pd(value2, &newSecond); - UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0); - UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0); -} - -void TSSEEmulTest::Test_mm_loadh_pd() { - const double first[2] = {1.3, 2.3}; - const double second[2] = {5.3, 6.3}; - const double firstEtalon[2] = {1.3, 10.13}; - const double secondEtalon[2] = {5.3, 11.13}; - - double newFirst = 10.13; - double newSecond = 11.13; - - __m128d value1 = _mm_loadu_pd(first); - __m128d value2 = _mm_loadu_pd(second); - value1 = _mm_loadh_pd(value1, &newFirst); - value2 = _mm_loadh_pd(value2, &newSecond); - UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0); - UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0); -} - +void TSSEEmulTest::Test_mm_shuffle_pd() { + const double first[2] = {1.3, 2.3}; + const double second[2] = {5.3, 6.3}; + const double etalon0[2] = {1.3, 5.3}; + const double etalon1[2] = {2.3, 5.3}; + const double etalon2[2] = {1.3, 6.3}; + const double etalon3[2] = {2.3, 6.3}; + + const __m128d value1 = _mm_loadu_pd(first); + const __m128d value2 = _mm_loadu_pd(second); + + __m128d res = _mm_shuffle_pd(value1, value2, 0); + UNIT_ASSERT_EQUAL(::memcmp(&res, etalon0, sizeof(etalon0)), 0); + + res = _mm_shuffle_pd(value1, value2, 1); + UNIT_ASSERT_EQUAL(::memcmp(&res, etalon1, sizeof(etalon1)), 0); + + res = _mm_shuffle_pd(value1, value2, 2); + UNIT_ASSERT_EQUAL(::memcmp(&res, etalon2, sizeof(etalon2)), 0); + + res = _mm_shuffle_pd(value1, value2, 3); + UNIT_ASSERT_EQUAL(::memcmp(&res, etalon3, sizeof(etalon3)), 0); +} + +void TSSEEmulTest::Test_mm_cvtsd_f64() { + const double first[2] = {1.3, 2.3}; + const double second[2] = {5.3, 6.3}; + + const __m128d value1 = _mm_loadu_pd(first); + const __m128d value2 = _mm_loadu_pd(second); + + UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value1), 1.3); + UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value2), 5.3); +} + +void TSSEEmulTest::Test_mm_loadl_pd() { + const double first[2] = {1.3, 2.3}; + const double second[2] = {5.3, 6.3}; + const double firstEtalon[2] = {10.13, 2.3}; + const double secondEtalon[2] = {11.13, 6.3}; + + double newFirst = 10.13; + double newSecond = 11.13; + + __m128d value1 = _mm_loadu_pd(first); + __m128d value2 = _mm_loadu_pd(second); + value1 = _mm_loadl_pd(value1, &newFirst); + value2 = _mm_loadl_pd(value2, &newSecond); + UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0); + UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0); +} + +void TSSEEmulTest::Test_mm_loadh_pd() { + const double first[2] = {1.3, 2.3}; + const double second[2] = {5.3, 6.3}; + const double firstEtalon[2] = {1.3, 10.13}; + const double secondEtalon[2] = {5.3, 11.13}; + + double newFirst = 10.13; + double newSecond = 11.13; + + __m128d value1 = _mm_loadu_pd(first); + __m128d value2 = _mm_loadu_pd(second); + value1 = _mm_loadh_pd(value1, &newFirst); + value2 = _mm_loadh_pd(value2, &newSecond); + UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0); + UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0); +} + void TSSEEmulTest::Test_mm_or_ps() { alignas(16) const char bytes1[16] = { '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', @@ -1990,99 +1990,99 @@ void TSSEEmulTest::Test_mm_or_ps() { UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); } - -void TSSEEmulTest::Test_mm_loadu_pd() { - alignas(16) double stub[4] = { - 0.f, 1.f, - 2.f, 3.f - }; - - for (size_t shift = 0; shift != 3; ++shift) { - const __m128d val = _mm_loadu_pd(&stub[shift]); - alignas(16) double res[2]; - _mm_store_pd(res, val); - - for (size_t i = 0; i != 2; ++i) { - UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16)); - } - } -} - -void TSSEEmulTest::Test_mm_rsqrt_ps() { - alignas(16) const char bytes[16] = { - '\x00', '\x00', '\x28', '\x42', // 42.f - '\x00', '\x98', '\x84', '\x45', // 4243.f - '\x60', '\x26', '\xcf', '\x48', // 424243.f - '\xed', '\xd5', '\x21', '\x4c' // 42424243.f - }; - const __m128 value = _mm_loadu_ps((const float*)bytes); - const __m128 result = _mm_rsqrt_ps(value); - alignas(16) float res[4]; - _mm_store_ps(res, result); - float fResult = 0.f; - for (size_t i = 0; i < 4; ++i) { - memcpy(&fResult, &bytes[i * 4], 4); - fResult = 1.f / std::sqrt(fResult); - UNIT_ASSERT_DOUBLES_EQUAL_C(res[i], fResult, 1e-3, "res: " << fResult << " vs etalon " << res[i]); - } -} - -namespace NHelpers { - - static __m128i Y_FORCE_INLINE GetCmp16(const __m128 &c0, const __m128 &c1, const __m128 &c2, const __m128 &c3, const __m128 test) { - const __m128i r0 = _mm_castps_si128(_mm_cmpgt_ps(c0, test)); - const __m128i r1 = _mm_castps_si128(_mm_cmpgt_ps(c1, test)); - const __m128i r2 = _mm_castps_si128(_mm_cmpgt_ps(c2, test)); - const __m128i r3 = _mm_castps_si128(_mm_cmpgt_ps(c3, test)); - const __m128i packed = _mm_packs_epi16(_mm_packs_epi32(r0, r1), _mm_packs_epi32(r2, r3)); - return _mm_and_si128(_mm_set1_epi8(0x01), packed); - } - - static __m128i Y_FORCE_INLINE GetCmp16(const float *factors, const __m128 test) { - const __m128 *ptr = (__m128 *)factors; - return GetCmp16(ptr[0], ptr[1], ptr[2], ptr[3], test); - } - - template<size_t Num> - void DoLane(size_t length, const float *factors, ui32 *& dst, const float *&values) { - for (size_t i = 0; i < length; ++i) { - __m128 value = _mm_set1_ps(values[i]); - __m128i agg = GetCmp16(factors, value); - if (Num > 1) { - agg = _mm_add_epi16(agg, _mm_slli_epi16(GetCmp16(&factors[64], value), 1)); - } - _mm_store_si128((__m128i *)&dst[4 * i], agg); - } - } -} - -void TSSEEmulTest::Test_matrixnet_powerpc() { - static constexpr size_t length = 10; - alignas(16) float factors[1024]; - alignas(16) ui32 valP[4 * length] = { 0 }; - float values[length]; - TReallyFastRng32 rng(42); - for (size_t i = 0; i < 1024; ++i) { - factors[i] = rng.GenRandReal2(); - } - for (size_t i = 0; i < length; ++i) { - values[i] = rng.GenRandReal2(); - } - ui32* val = reinterpret_cast<ui32*>(valP); - const float* vals = reinterpret_cast<const float*>(values); - NHelpers::DoLane<2>(length, factors, val, vals); - static const ui32 etalon[4 * length] = { - 2, 33554432, 258, 33554433, 50529027, - 50529027, 50529027, 50529027, 50528770, - 33685763, 33555203, 50462723, 50528770, - 33685763, 33555203, 50462723, 50529026, - 33751299, 50529027, 50463491, 2, 33554432, - 258, 33554433, 50397698, 33685761, 259, - 50462721, 50332162, 33554689, 259, 50462721, - 50528770, 33685761, 33555203, 50462723, - 50529026, 33685763, 50463491, 50463235 - }; - for (size_t i = 0; i < 4 * length; ++i) { - UNIT_ASSERT_EQUAL(valP[i], etalon[i]); - } -} + +void TSSEEmulTest::Test_mm_loadu_pd() { + alignas(16) double stub[4] = { + 0.f, 1.f, + 2.f, 3.f + }; + + for (size_t shift = 0; shift != 3; ++shift) { + const __m128d val = _mm_loadu_pd(&stub[shift]); + alignas(16) double res[2]; + _mm_store_pd(res, val); + + for (size_t i = 0; i != 2; ++i) { + UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16)); + } + } +} + +void TSSEEmulTest::Test_mm_rsqrt_ps() { + alignas(16) const char bytes[16] = { + '\x00', '\x00', '\x28', '\x42', // 42.f + '\x00', '\x98', '\x84', '\x45', // 4243.f + '\x60', '\x26', '\xcf', '\x48', // 424243.f + '\xed', '\xd5', '\x21', '\x4c' // 42424243.f + }; + const __m128 value = _mm_loadu_ps((const float*)bytes); + const __m128 result = _mm_rsqrt_ps(value); + alignas(16) float res[4]; + _mm_store_ps(res, result); + float fResult = 0.f; + for (size_t i = 0; i < 4; ++i) { + memcpy(&fResult, &bytes[i * 4], 4); + fResult = 1.f / std::sqrt(fResult); + UNIT_ASSERT_DOUBLES_EQUAL_C(res[i], fResult, 1e-3, "res: " << fResult << " vs etalon " << res[i]); + } +} + +namespace NHelpers { + + static __m128i Y_FORCE_INLINE GetCmp16(const __m128 &c0, const __m128 &c1, const __m128 &c2, const __m128 &c3, const __m128 test) { + const __m128i r0 = _mm_castps_si128(_mm_cmpgt_ps(c0, test)); + const __m128i r1 = _mm_castps_si128(_mm_cmpgt_ps(c1, test)); + const __m128i r2 = _mm_castps_si128(_mm_cmpgt_ps(c2, test)); + const __m128i r3 = _mm_castps_si128(_mm_cmpgt_ps(c3, test)); + const __m128i packed = _mm_packs_epi16(_mm_packs_epi32(r0, r1), _mm_packs_epi32(r2, r3)); + return _mm_and_si128(_mm_set1_epi8(0x01), packed); + } + + static __m128i Y_FORCE_INLINE GetCmp16(const float *factors, const __m128 test) { + const __m128 *ptr = (__m128 *)factors; + return GetCmp16(ptr[0], ptr[1], ptr[2], ptr[3], test); + } + + template<size_t Num> + void DoLane(size_t length, const float *factors, ui32 *& dst, const float *&values) { + for (size_t i = 0; i < length; ++i) { + __m128 value = _mm_set1_ps(values[i]); + __m128i agg = GetCmp16(factors, value); + if (Num > 1) { + agg = _mm_add_epi16(agg, _mm_slli_epi16(GetCmp16(&factors[64], value), 1)); + } + _mm_store_si128((__m128i *)&dst[4 * i], agg); + } + } +} + +void TSSEEmulTest::Test_matrixnet_powerpc() { + static constexpr size_t length = 10; + alignas(16) float factors[1024]; + alignas(16) ui32 valP[4 * length] = { 0 }; + float values[length]; + TReallyFastRng32 rng(42); + for (size_t i = 0; i < 1024; ++i) { + factors[i] = rng.GenRandReal2(); + } + for (size_t i = 0; i < length; ++i) { + values[i] = rng.GenRandReal2(); + } + ui32* val = reinterpret_cast<ui32*>(valP); + const float* vals = reinterpret_cast<const float*>(values); + NHelpers::DoLane<2>(length, factors, val, vals); + static const ui32 etalon[4 * length] = { + 2, 33554432, 258, 33554433, 50529027, + 50529027, 50529027, 50529027, 50528770, + 33685763, 33555203, 50462723, 50528770, + 33685763, 33555203, 50462723, 50529026, + 33751299, 50529027, 50463491, 2, 33554432, + 258, 33554433, 50397698, 33685761, 259, + 50462721, 50332162, 33554689, 259, 50462721, + 50528770, 33685761, 33555203, 50462723, + 50529026, 33685763, 50463491, 50463235 + }; + for (size_t i = 0; i < 4 * length; ++i) { + UNIT_ASSERT_EQUAL(valP[i], etalon[i]); + } +} diff --git a/library/cpp/sse/ut/ya.make b/library/cpp/sse/ut/ya.make index 3605e95fad..45e104971e 100644 --- a/library/cpp/sse/ut/ya.make +++ b/library/cpp/sse/ut/ya.make @@ -1,13 +1,13 @@ UNITTEST_FOR(library/cpp/sse) -OWNER(danlark) +OWNER(danlark) SRCS( test.cpp ) -IF (ARCH_X86_64) - CFLAGS(-msse4.1 -msse4.2) -ENDIF() - +IF (ARCH_X86_64) + CFLAGS(-msse4.1 -msse4.2) +ENDIF() + END() diff --git a/library/cpp/sse/ya.make b/library/cpp/sse/ya.make index f37918aced..d2351e791d 100644 --- a/library/cpp/sse/ya.make +++ b/library/cpp/sse/ya.make @@ -1,12 +1,12 @@ LIBRARY() -OWNER( - g:base - danlark -) +OWNER( + g:base + danlark +) SRCS( - sse.cpp + sse.cpp ) END() |