aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/sse
diff options
context:
space:
mode:
authorDevtools Arcadia <arcadia-devtools@yandex-team.ru>2022-02-07 18:08:42 +0300
committerDevtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net>2022-02-07 18:08:42 +0300
commit1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch)
treee26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/sse
downloadydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/sse')
-rw-r--r--library/cpp/sse/README.md7
-rw-r--r--library/cpp/sse/powerpc.h1000
-rw-r--r--library/cpp/sse/sse.cpp1
-rw-r--r--library/cpp/sse/sse.h33
-rw-r--r--library/cpp/sse/sse2neon.h1045
-rw-r--r--library/cpp/sse/ut/test.cpp2088
-rw-r--r--library/cpp/sse/ut/ya.make13
-rw-r--r--library/cpp/sse/ya.make12
8 files changed, 4199 insertions, 0 deletions
diff --git a/library/cpp/sse/README.md b/library/cpp/sse/README.md
new file mode 100644
index 0000000000..a82c98a1b9
--- /dev/null
+++ b/library/cpp/sse/README.md
@@ -0,0 +1,7 @@
+Overview
+===
+This library provides the implementation of Intel SSE intrinsics for other CPU architectures. Currently supports PowerPC via translation to AltiVec and ARM via NEON. In some cases, falls back to software emulation if there's no corresponding instruction in the target instruction set.
+
+Usage
+===
+Include library/cpp/sse/sse.h and use the needed intrinsics. Implementation will be selected based on the target architecture of the used toolchain.
diff --git a/library/cpp/sse/powerpc.h b/library/cpp/sse/powerpc.h
new file mode 100644
index 0000000000..82fc011fa5
--- /dev/null
+++ b/library/cpp/sse/powerpc.h
@@ -0,0 +1,1000 @@
+#pragma once
+
+/*
+ The header contains code which translates SSE intrinsics
+ to PowerPC AltiVec or software emulation.
+
+ See also: https://www.ibm.com/developerworks/community/wikis/home?lang=en#!/wiki/W51a7ffcf4dfd_4b40_9d82_446ebc23c550/page/Intel%20SSE%20to%20PowerPC%20AltiVec%20migration
+*/
+/* Author: Vadim Rumyantsev <rumvadim@yandex-team.ru> */
+
+#if !defined(_ppc64_)
+#error "This header is for PowerPC (ppc64) platform only." \
+ "Include sse.h instead of including this header directly."
+#endif
+
+#include <util/system/types.h>
+#include <util/system/compiler.h>
+
+#include <altivec.h>
+
+typedef __attribute__((__aligned__(8))) unsigned long long __m64;
+typedef __attribute__((__aligned__(16), __may_alias__)) vector float __m128;
+typedef __attribute__((__aligned__(16), __may_alias__)) vector unsigned char __m128i;
+typedef __attribute__((__aligned__(16), __may_alias__)) vector double __m128d;
+
+using __v2df = __vector double;
+using __v2di = __vector long long;
+using __v2du = __vector unsigned long long;
+using __v4si = __vector int;
+using __v4su = __vector unsigned int;
+using __v8hi = __vector short;
+using __v8hu = __vector unsigned short;
+using __v16qi = __vector signed char;
+using __v16qu = __vector unsigned char;
+using __v4sf = __vector float;
+
+enum _mm_hint
+{
+ /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */
+ _MM_HINT_ET0 = 7,
+ _MM_HINT_ET1 = 6,
+ _MM_HINT_T0 = 3,
+ _MM_HINT_T1 = 2,
+ _MM_HINT_T2 = 1,
+ _MM_HINT_NTA = 0
+};
+
+#define _MM_SHUFFLE(a, b, c, d) ((signed char)(a * 64 + b * 16 + c * 4 + d))
+
+/// Functions that work with floats.
+
+Y_FORCE_INLINE __m128 _mm_setzero_ps() {
+ return (__m128){0.0f, 0.0f, 0.0f, 0.0f};
+};
+
+Y_FORCE_INLINE __m128d _mm_setzero_pd() {
+ return (__m128d)vec_splats((double)0);
+}
+
+// bug in clang compiler until 7.0.0 inclusive, Y_NO_INLINE is vital/essential
+static Y_NO_INLINE __m128 _mm_set1_ps(float f) {
+ return (vector float)f;
+}
+
+Y_FORCE_INLINE __m128 _mm_set_ps1(float f) {
+ return _mm_set1_ps(f);
+}
+
+Y_FORCE_INLINE __m128 _mm_set_ps(float v3, float v2, float v1, float v0) {
+ return (__m128)(__v4sf){v0, v1, v2, v3};
+}
+
+Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) {
+ return (__m128d){d0, d1};
+}
+
+Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* p) {
+ return vec_vsx_ld(0, p);
+}
+
+Y_FORCE_INLINE __m128 _mm_load_ps(const float* p) {
+ return (__m128)vec_ld(0, (vector float*)p);
+}
+
+Y_FORCE_INLINE __m128 _mm_loadu_pd(const double* d) {
+ return vec_vsx_ld(0, d);
+}
+
+Y_FORCE_INLINE void _mm_storeu_ps(float* p, __m128 a) {
+ *(__m128*)p = a;
+}
+
+Y_FORCE_INLINE __m128 _mm_xor_ps(__m128 a, __m128 b) {
+ return (__m128)vec_xor((__v4sf)a, (__v4sf)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_xor_pd(__m128d a, __m128d b) {
+ return (__m128)vec_xor((__v2df)a, (__v2df)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_add_ps(__m128 a, __m128 b) {
+ return (__m128)((__v4sf)a + (__v4sf)b);
+}
+
+Y_FORCE_INLINE __m128d _mm_add_pd(__m128d a, __m128d b) {
+ return (__m128d)((__v2df)a + (__v2df)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_sub_ps(__m128 a, __m128 b) {
+ return (__m128)((__v4sf)a - (__v4sf)b);
+}
+
+Y_FORCE_INLINE __m128d _mm_sub_pd(__m128d a, __m128d b) {
+ return (__m128d)((__v2df)a - (__v2df)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_mul_ps(__m128 a, __m128 b) {
+ return (__m128)((__v4sf)a * (__v4sf)b);
+}
+
+Y_FORCE_INLINE __m128d _mm_mul_pd(__m128d a, __m128d b) {
+ return (__m128d)((__v2df)a * (__v2df)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_div_ps(__m128 a, __m128 b) {
+ return (__m128)((__v4sf)a / (__v4sf)b);
+}
+
+Y_FORCE_INLINE __m128d _mm_div_pd(__m128d a, __m128d b) {
+ return (__m128d)((__v2df)a / (__v2df)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_cmpeq_ps(__m128 a, __m128 b) {
+ return ((__m128)vec_cmpeq((__v4sf)a, (__v4sf)b));
+ ;
+}
+
+Y_FORCE_INLINE __m128 _mm_cmpgt_ps(__m128 a, __m128 b) {
+ return ((__m128)vec_cmpgt((__v4sf)a, (__v4sf)b));
+}
+
+Y_FORCE_INLINE __m128 _mm_max_ps(__m128 a, __m128 b) {
+ return (__m128)vec_max((vector float)a, (vector float)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) {
+ return (__m128i)vec_max((__v16qu)a, (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_min_ps(__m128 a, __m128 b) {
+ return (__m128)vec_min((vector float)a, (vector float)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_and_ps(__m128 a, __m128 b) {
+ return ((__m128)vec_and((__v4sf)a, (__v4sf)b));
+}
+
+Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) {
+ return vec_and((__v2df)a, (__v2df)b);
+}
+
+Y_FORCE_INLINE __m128 _mm_rsqrt_ps(__m128 a) {
+ return vec_rsqrte(a);
+}
+
+Y_FORCE_INLINE __m128 _mm_rsqrt_ss(__m128 a) {
+ __m128 a1, c;
+ const vector unsigned int mask = {0xffffffff, 0, 0, 0};
+ a1 = vec_splat(a, 0);
+ c = vec_rsqrte(a1);
+ return (vec_sel((vector float)a, c, mask));
+}
+
+Y_FORCE_INLINE int _mm_movemask_ps(__m128 a) {
+ __vector unsigned long long result;
+ const __vector unsigned int perm_mask =
+ {
+#ifdef __LITTLE_ENDIAN__
+ 0x00204060, 0x80808080, 0x80808080, 0x80808080
+#elif __BIG_ENDIAN__
+ 0x80808080, 0x80808080, 0x80808080, 0x00204060
+#endif
+ };
+
+ result = (__vector unsigned long long)vec_vbpermq((__vector unsigned char)a,
+ (__vector unsigned char)perm_mask);
+
+#ifdef __LITTLE_ENDIAN__
+ return result[1];
+#elif __BIG_ENDIAN__
+ return result[0];
+#endif
+}
+
+Y_FORCE_INLINE __m128 _mm_cvtepi32_ps(__m128i a) {
+ return ((__m128)vec_ctf((__v4si)a, 0));
+}
+
+Y_FORCE_INLINE float _mm_cvtss_f32(__m128 a) {
+ return ((__v4sf)a)[0];
+}
+
+Y_FORCE_INLINE __m128 _mm_cmpunord_ps(__m128 A, __m128 B) {
+ __vector unsigned int a, b;
+ __vector unsigned int c, d;
+ const __vector unsigned int float_exp_mask =
+ {0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
+
+ a = (__vector unsigned int)vec_abs((__v4sf)A);
+ b = (__vector unsigned int)vec_abs((__v4sf)B);
+ c = (__vector unsigned int)vec_cmpgt(a, float_exp_mask);
+ d = (__vector unsigned int)vec_cmpgt(b, float_exp_mask);
+ return ((__m128)vec_or(c, d));
+}
+
+Y_FORCE_INLINE __m128 _mm_andnot_ps(__m128 a, __m128 b) {
+ return ((__m128)vec_andc((__v4sf)b, (__v4sf)a));
+}
+
+Y_FORCE_INLINE __m128 _mm_or_ps(__m128 a, __m128 b) {
+ return ((__m128)vec_or((__v4sf)a, (__v4sf)b));
+}
+
+Y_FORCE_INLINE void _mm_store_ss(float* p, __m128 a) {
+ *p = ((__v4sf)a)[0];
+}
+
+Y_FORCE_INLINE void _mm_store_ps(float* p, __m128 a) {
+ vec_st(a, 0, p);
+}
+
+Y_FORCE_INLINE void _mm_storeu_pd(double* p, __m128d a) {
+ *(__m128d*)p = a;
+}
+
+Y_FORCE_INLINE void _mm_store_pd(double* p, __m128d a) {
+ vec_st((vector unsigned char)a, 0, (vector unsigned char*)p);
+}
+
+Y_FORCE_INLINE __m128 _mm_shuffle_ps(__m128 a, __m128 b, long shuff) {
+ unsigned long element_selector_10 = shuff & 0x03;
+ unsigned long element_selector_32 = (shuff >> 2) & 0x03;
+ unsigned long element_selector_54 = (shuff >> 4) & 0x03;
+ unsigned long element_selector_76 = (shuff >> 6) & 0x03;
+ const unsigned int permute_selectors[4] =
+ {
+#ifdef __LITTLE_ENDIAN__
+ 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
+#elif __BIG_ENDIAN__
+ 0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203
+#endif
+ };
+ __vector unsigned int t;
+
+#ifdef __LITTLE_ENDIAN__
+ t[0] = permute_selectors[element_selector_10];
+ t[1] = permute_selectors[element_selector_32];
+ t[2] = permute_selectors[element_selector_54] + 0x10101010;
+ t[3] = permute_selectors[element_selector_76] + 0x10101010;
+#elif __BIG_ENDIAN__
+ t[3] = permute_selectors[element_selector_10] + 0x10101010;
+ t[2] = permute_selectors[element_selector_32] + 0x10101010;
+ t[1] = permute_selectors[element_selector_54];
+ t[0] = permute_selectors[element_selector_76];
+#endif
+ return vec_perm((__v4sf)a, (__v4sf)b, (__vector unsigned char)t);
+}
+
+Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, const int mask) {
+ __vector double result;
+ const int litmsk = mask & 0x3;
+
+ if (litmsk == 0)
+ result = vec_mergeh(a, b);
+ else if (litmsk == 1)
+ result = vec_xxpermdi(a, b, 2);
+ else if (litmsk == 2)
+ result = vec_xxpermdi(a, b, 1);
+ else
+ result = vec_mergel(a, b);
+ return result;
+}
+
+Y_FORCE_INLINE __m128i _mm_cvtps_epi32(__m128 a) {
+ vector float rounded;
+ __v4si result;
+
+ rounded = vec_rint((vector float)a);
+ result = vec_cts(rounded, 0);
+ return (__m128i)result;
+}
+
+/// Functions that work with integers.
+
+Y_FORCE_INLINE int _mm_movemask_epi8(__m128i a) {
+ __vector unsigned long long result;
+ const __vector unsigned char perm_mask =
+ {
+#ifdef __LITTLE_ENDIAN__
+ 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40,
+ 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00
+#elif __BIG_ENDIAN__
+ 0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38,
+ 0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78
+#endif
+ };
+
+ result = (__vector unsigned long long)vec_vbpermq((__vector unsigned char)a,
+ (__vector unsigned char)perm_mask);
+
+#ifdef __LITTLE_ENDIAN__
+ return result[1];
+#elif __BIG_ENDIAN__
+ return result[0];
+#endif
+}
+
+Y_FORCE_INLINE __m128i _mm_cvttps_epi32(__m128 a) {
+ __v4si result;
+
+ result = vec_cts((__v4sf)a, 0);
+ return (__m128i)result;
+}
+
+#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \
+ do { \
+ __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \
+ __v4sf __t0 = vec_vmrghw((vector unsigned int)__r0, (vector unsigned int)__r1); \
+ __v4sf __t1 = vec_vmrghw((vector unsigned int)__r2, (vector unsigned int)__r3); \
+ __v4sf __t2 = vec_vmrglw((vector unsigned int)__r0, (vector unsigned int)__r1); \
+ __v4sf __t3 = vec_vmrglw((vector unsigned int)__r2, (vector unsigned int)__r3); \
+ (row0) = (__v4sf)vec_mergeh((vector long long)__t0, \
+ (vector long long)__t1); \
+ (row1) = (__v4sf)vec_mergel((vector long long)__t0, \
+ (vector long long)__t1); \
+ (row2) = (__v4sf)vec_mergeh((vector long long)__t2, \
+ (vector long long)__t3); \
+ (row3) = (__v4sf)vec_mergel((vector long long)__t2, \
+ (vector long long)__t3); \
+ } while (0)
+
+Y_FORCE_INLINE __m128i _mm_or_si128(__m128i a, __m128i b) {
+ return (__m128i)vec_or((__v2di)a, (__v2di)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_and_si128(__m128i a, __m128i b) {
+ return (__m128i)vec_and((__v2di)a, (__v2di)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_andnot_si128(__m128i a, __m128i b) {
+ return (__m128i)vec_andc((__v2di)b, (__v2di)a);
+}
+
+Y_FORCE_INLINE __m128i _mm_xor_si128(__m128i a, __m128i b) {
+ return (__m128i)vec_xor((__v2di)a, (__v2di)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_setzero_si128() {
+ return (__m128i)(__v4si){0, 0, 0, 0};
+}
+
+Y_FORCE_INLINE __m128i _mm_shuffle_epi32(__m128i op1, long op2) {
+ unsigned long element_selector_10 = op2 & 0x03;
+ unsigned long element_selector_32 = (op2 >> 2) & 0x03;
+ unsigned long element_selector_54 = (op2 >> 4) & 0x03;
+ unsigned long element_selector_76 = (op2 >> 6) & 0x03;
+ const unsigned int permute_selectors[4] =
+ {
+#ifdef __LITTLE_ENDIAN__
+ 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
+#elif __BIG_ENDIAN__
+ 0x0C0D0E0F, 0x08090A0B, 0x04050607, 0x00010203
+#endif
+ };
+ __v4su t;
+
+#ifdef __LITTLE_ENDIAN__
+ t[0] = permute_selectors[element_selector_10];
+ t[1] = permute_selectors[element_selector_32];
+ t[2] = permute_selectors[element_selector_54] + 0x10101010;
+ t[3] = permute_selectors[element_selector_76] + 0x10101010;
+#elif __BIG_ENDIAN__
+ t[3] = permute_selectors[element_selector_10] + 0x10101010;
+ t[2] = permute_selectors[element_selector_32] + 0x10101010;
+ t[1] = permute_selectors[element_selector_54];
+ t[0] = permute_selectors[element_selector_76];
+#endif
+ return (__m128i)vec_perm((__v4si)op1, (__v4si)op1, (__vector unsigned char)t);
+}
+
+Y_FORCE_INLINE int _mm_extract_epi16(__m128i a, int imm) {
+ return (unsigned short)((__v8hi)a)[imm & 7];
+}
+
+Y_FORCE_INLINE int _mm_extract_epi8(__m128i a, int imm) {
+ return (unsigned char)((__v16qi)a)[imm & 15];
+}
+
+Y_FORCE_INLINE int _mm_extract_epi32(__m128i a, int imm) {
+ return ((__v4si)a)[imm & 3];
+}
+
+Y_FORCE_INLINE long long _mm_extract_epi64(__m128i a, int imm) {
+ return ((__v2di)a)[imm & 1];
+}
+
+Y_FORCE_INLINE int _mm_extract_ps(__m128 a, int imm) {
+ return ((__v4si)a)[imm & 3];
+}
+
+Y_FORCE_INLINE __m128i _mm_slli_epi16(__m128i a, int count) {
+ __v8hu lshift;
+ __v8hi result = {0, 0, 0, 0, 0, 0, 0, 0};
+
+ if (count >= 0 && count < 16) {
+ if (__builtin_constant_p(count)) {
+ lshift = (__v8hu)vec_splat_s16(count);
+ } else {
+ lshift = vec_splats((unsigned short)count);
+ }
+
+ result = vec_vslh((__v8hi)a, lshift);
+ }
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_slli_epi32(__m128i a, int count) {
+ __v4su lshift;
+ __v4si result = {0, 0, 0, 0};
+
+ if (count >= 0 && count < 32) {
+ if (__builtin_constant_p(count) && count < 16) {
+ lshift = (__v4su)vec_splat_s32(count);
+ } else {
+ lshift = vec_splats((unsigned int)count);
+ }
+
+ result = vec_vslw((__v4si)a, lshift);
+ }
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_slli_epi64(__m128i a, int count) {
+ __v2du lshift;
+ __v2di result = {0, 0};
+
+ if (count >= 0 && count < 64) {
+ if (__builtin_constant_p(count) && count < 16) {
+ lshift = (__v2du)vec_splat_s32(count);
+ } else {
+ lshift = (__v2du)vec_splats((unsigned int)count);
+ }
+
+ result = vec_sl((__v2di)a, lshift);
+ }
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_slli_si128(__m128i a, int imm) {
+ __v16qu result;
+ const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+ if (imm < 16)
+#ifdef __LITTLE_ENDIAN__
+ result = vec_sld((__v16qu)a, zeros, imm);
+#elif __BIG_ENDIAN__
+ result = vec_sld(zeros, (__v16qu)a, (16 - imm));
+#endif
+ else
+ result = zeros;
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_srli_epi16(__m128i a, int count) {
+ if ((unsigned long)count >= 16) {
+ /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */
+ return (__m128i)vec_splats(0);
+ } else if (count == 0) {
+ return a;
+ } else {
+ /* The PowerPC Architecture says all shift count fields must contain the same shift count. */
+ __v8hi replicated_count;
+ replicated_count = vec_splats((short)count);
+ return (__m128i)vec_sr((vector signed short)a, replicated_count);
+ }
+}
+
+Y_FORCE_INLINE __m128i _mm_srli_epi32(__m128i a, int count) {
+ if ((unsigned long)count >= 32) {
+ /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */
+ return (__m128i)vec_splats(0);
+ } else if (count == 0) {
+ return a;
+ } else {
+ /* The PowerPC Architecture says all shift count fields must contain the same shift count. */
+ __v4si replicated_count;
+ replicated_count = vec_splats(count);
+ return (__m128i)vec_sr((vector signed int)a, replicated_count);
+ }
+}
+
+Y_FORCE_INLINE __m128i _mm_srli_epi64(__m128i a, int count) {
+ if ((unsigned long)count >= 64) {
+ /* SSE2 shifts >= element_size or < 0 produce 0; Altivec/MMX shifts by count%element_size. */
+ return (__m128i)vec_splats(0);
+ } else if (count == 0) {
+ return a;
+ } else {
+ /* The PowerPC Architecture says all shift count fields must contain the same shift count. */
+ /* On Power7 vec_slo (vslo) does use just the documented bits 121:124. */
+ /* On Power7 vec_sll (vsll) uses the lower 3 bits of each byte instead (legal). */
+ __v16qu replicated_count;
+ replicated_count = vec_splats((unsigned char)count);
+ long long m = 0xFFFFFFFFFFFFFFFFull >> count;
+ __v2di mask;
+ mask[0] = m;
+ mask[1] = m;
+ return vec_and(vec_srl(vec_sro(a, (__m128i)replicated_count), (__m128i)replicated_count), (__v16qu)mask);
+ }
+}
+
+Y_FORCE_INLINE __m128i _mm_bsrli_si128(__m128i a, const int __N) {
+ __v16qu result;
+ const __v16qu zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+ if (__N < 16)
+ if (__builtin_constant_p(__N))
+ /* Would like to use Vector Shift Left Double by Octet
+ Immediate here to use the immediate form and avoid
+ load of __N * 8 value into a separate VR. */
+ result = vec_sld(zeros, (__v16qu)a, (16 - __N));
+ else {
+ __v16qu shift = vec_splats((unsigned char)(__N * 8));
+ result = vec_sro((__v16qu)a, shift);
+ }
+ else
+ result = zeros;
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_srli_si128(__m128i a, int imm) {
+ return _mm_bsrli_si128(a, imm);
+}
+
+Y_FORCE_INLINE __m128i _mm_srai_epi16(__m128i a, int count) {
+ __v8hu rshift = {15, 15, 15, 15, 15, 15, 15, 15};
+ __v8hi result;
+ if (count < 16) {
+ if (__builtin_constant_p(count)) {
+ rshift = (__v8hu)vec_splat_s16(count);
+ } else {
+ rshift = vec_splats((unsigned short)count);
+ }
+ }
+ result = vec_vsrah((__v8hi)a, rshift);
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_srai_epi32(__m128i a, int count) {
+ // return vec_shiftrightarithmetic4wimmediate(a, count); //!< Failes to work with count >= 32.
+ __v4su rshift = {31, 31, 31, 31};
+ __v4si result;
+
+ if (count < 32) {
+ if (__builtin_constant_p(count)) {
+ if (count < 16) {
+ rshift = (__v4su)vec_splat_s32(count);
+ } else {
+ rshift = (__v4su)vec_splats((unsigned int)count);
+ }
+ } else {
+ rshift = vec_splats((unsigned int)count);
+ }
+ }
+ result = vec_vsraw((__v4si)a, rshift);
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_sll_epi16(__m128i a, __m128i count) {
+ __v8hu lshift, shmask;
+ const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15};
+ __v8hu result;
+
+#ifdef __LITTLE_ENDIAN__
+ lshift = vec_splat((__v8hu)count, 0);
+#elif __BIG_ENDIAN__
+ lshift = vec_splat((__v8hu)count, 3);
+#endif
+ shmask = vec_cmple(lshift, shmax);
+ result = vec_vslh((__v8hu)a, lshift);
+ result = vec_sel(shmask, result, shmask);
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_sll_epi32(__m128i a, __m128i count) {
+ __v4su lshift, shmask;
+ const __v4su shmax = {32, 32, 32, 32};
+ __v4su result;
+#ifdef __LITTLE_ENDIAN__
+ lshift = vec_splat((__v4su)count, 0);
+#elif __BIG_ENDIAN__
+ lshift = vec_splat((__v4su)count, 1);
+#endif
+ shmask = vec_cmplt(lshift, shmax);
+ result = vec_vslw((__v4su)a, lshift);
+ result = vec_sel(shmask, result, shmask);
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_sll_epi64(__m128i a, __m128i count) {
+ __v2du lshift, shmask;
+ const __v2du shmax = {64, 64};
+ __v2du result;
+
+ lshift = (__v2du)vec_splat((__v2du)count, 0);
+ shmask = vec_cmplt(lshift, shmax);
+ result = vec_sl((__v2du)a, lshift);
+ result = ((vector long long)shmask & ~(vector long long)shmask) | ((vector long long)result & (vector long long)shmask);
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_srl_epi16(__m128i a, __m128i count) {
+ __v8hu rshift, shmask;
+ const __v8hu shmax = {15, 15, 15, 15, 15, 15, 15, 15};
+ __v8hu result;
+
+#ifdef __LITTLE_ENDIAN__
+ rshift = vec_splat((__v8hu)count, 0);
+#elif __BIG_ENDIAN__
+ rshift = vec_splat((__v8hu)count, 3);
+#endif
+ shmask = vec_cmple(rshift, shmax);
+ result = vec_vsrh((__v8hu)a, rshift);
+ result = vec_sel(shmask, result, shmask);
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_srl_epi32(__m128i a, __m128i count) {
+ __v4su rshift, shmask;
+ const __v4su shmax = {32, 32, 32, 32};
+ __v4su result;
+
+#ifdef __LITTLE_ENDIAN__
+ rshift = vec_splat((__v4su)count, 0);
+#elif __BIG_ENDIAN__
+ rshift = vec_splat((__v4su)count, 1);
+#endif
+ shmask = vec_cmplt(rshift, shmax);
+ result = vec_vsrw((__v4su)a, rshift);
+ result = vec_sel(shmask, result, shmask);
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_srl_epi64(__m128i a, __m128i count) {
+ __v2du rshift, shmask;
+ const __v2du shmax = {64, 64};
+ __v2du result;
+
+ rshift = (__v2du)vec_splat((__v2du)count, 0);
+ shmask = vec_cmplt(rshift, shmax);
+ result = vec_sr((__v2du)a, rshift);
+ result = (__v2du)vec_sel((__v2du)shmask, (__v2du)result, (__v2du)shmask);
+
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE void _mm_storeu_si128(__m128i* p, __m128i a) {
+ vec_vsx_st(a, 0, p);
+}
+
+Y_FORCE_INLINE void _mm_store_si128(__m128i* p, __m128i a) {
+ vec_st((__v16qu)a, 0, (__v16qu*)p);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpackhi_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_mergel((__v16qu)a, (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpackhi_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_mergel((__v8hu)a, (__v8hu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpackhi_epi32(__m128i a, __m128i b) {
+ return (__m128i)vec_mergel((__v4su)a, (__v4su)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpackhi_epi64(__m128i a, __m128i b) {
+ return (__m128i)vec_mergel((vector long long)a, (vector long long)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpacklo_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_mergeh((__v16qu)a, (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpacklo_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_mergeh((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpacklo_epi32(__m128i a, __m128i b) {
+ return (__m128i)vec_mergeh((__v4si)a, (__v4si)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_unpacklo_epi64(__m128i a, __m128i b) {
+ return (__m128i)vec_mergeh((vector long long)a, (vector long long)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_add_epi8(__m128i a, __m128i b) {
+ return (__m128i)((__v16qu)a + (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_add_epi16(__m128i a, __m128i b) {
+ return (__m128i)((__v8hu)a + (__v8hu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_add_epi32(__m128i a, __m128i b) {
+ return (__m128i)((__v4su)a + (__v4su)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_add_epi64(__m128i a, __m128i b) {
+ return (__m128i)((__v2du)a + (__v2du)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_madd_epi16(__m128i a, __m128i b) {
+ const vector signed int zero = {0, 0, 0, 0};
+ return (__m128i)vec_vmsumshm((__v8hi)a, (__v8hi)b, zero);
+}
+
+Y_FORCE_INLINE __m128i _mm_sub_epi8(__m128i a, __m128i b) {
+ return (__m128i)((__v16qu)a - (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_sub_epi16(__m128i a, __m128i b) {
+ return (__m128i)((__v8hu)a - (__v8hu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_sub_epi32(__m128i a, __m128i b) {
+ return (__m128i)((__v4su)a - (__v4su)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_sub_epi64(__m128i a, __m128i b) {
+ return (__m128i)((__v2du)a - (__v2du)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_mul_epu32(__m128i a, __m128 b) {
+#ifdef __LITTLE_ENDIAN__
+ return (__m128i)vec_mule((__v4su)a, (__v4su)b);
+#elif __BIG_ENDIAN__
+ return (__m128i)vec_mulo((__v4su)a, (__v4su)b);
+#endif
+}
+
+Y_FORCE_INLINE __m128i _mm_set_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) {
+ return (__m128i)(__v16qi){q00, q01, q02, q03, q04, q05, q06, q07, q08, q09, q10, q11, q12, q13, q14, q15};
+};
+
+Y_FORCE_INLINE __m128i _mm_setr_epi8(char q15, char q14, char q13, char q12, char q11, char q10, char q09, char q08, char q07, char q06, char q05, char q04, char q03, char q02, char q01, char q00) {
+ return (__m128i)(__v16qi){q15, q14, q13, q12, q11, q10, q09, q08, q07, q06, q05, q04, q03, q02, q01, q00};
+};
+
+Y_FORCE_INLINE __m128i _mm_set_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) {
+ return (__m128i)(__v8hi){q0, q1, q2, q3, q4, q5, q6, q7};
+}
+
+Y_FORCE_INLINE __m128i _mm_setr_epi16(short q7, short q6, short q5, short q4, short q3, short q2, short q1, short q0) {
+ return (__m128i)(__v8hi){q7, q6, q5, q4, q3, q2, q1, q0};
+}
+
+Y_FORCE_INLINE __m128i _mm_set_epi32(int q3, int q2, int q1, int q0) {
+ return (__m128i)(__v4si){q0, q1, q2, q3};
+}
+
+Y_FORCE_INLINE __m128i _mm_setr_epi32(int q3, int q2, int q1, int q0) {
+ return (__m128i)(__v4si){q3, q2, q1, q0};
+}
+
+Y_FORCE_INLINE __m128i _mm_set1_epi8(char a) {
+ return _mm_set_epi8(a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a);
+}
+
+Y_FORCE_INLINE __m128i _mm_set1_epi16(short a) {
+ return _mm_set_epi16(a, a, a, a, a, a, a, a);
+}
+
+Y_FORCE_INLINE __m128i _mm_set1_epi32(int a) {
+ return _mm_set_epi32(a, a, a, a);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_cmpeq((__v16qi)a, (__v16qi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpeq_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_cmpeq((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpeq_epi32(__m128i a, __m128i b) {
+ return (__m128i)vec_cmpeq((__v4si)a, (__v4si)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_packs_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_packs((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_packs_epi32(__m128i a, __m128i b) {
+ return (__m128i)vec_packs((__v4si)a, (__v4si)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_packus_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_packsu((vector signed short)a, (vector signed short)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cvtsi64_si128(i64 a) {
+ return (__m128i)(__v2di){a, 0LL};
+}
+
+Y_FORCE_INLINE __m128i _mm_cvtsi32_si128(int a) {
+ return _mm_set_epi32(0, 0, 0, a);
+}
+
+Y_FORCE_INLINE int _mm_cvtsi128_si32(__m128i a) {
+ return ((__v4si)a)[0];
+}
+
+Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) {
+ return ((__v2di)a)[0];
+}
+
+Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* p) {
+ return *p;
+}
+
+Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* p) {
+ return (__m128i)(vec_vsx_ld(0, (signed int const*)p));
+}
+
+Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* p) {
+ return _mm_loadu_si128(p);
+}
+
+Y_FORCE_INLINE __m128i _mm_loadl_epi64(const __m128i* a) {
+#ifdef __LITTLE_ENDIAN__
+ const vector bool long long mask = {
+ 0xFFFFFFFFFFFFFFFFull, 0x0000000000000000ull};
+#elif __BIG_ENDIAN__
+ const vector bool long long mask = {
+ 0x0000000000000000ull, 0xFFFFFFFFFFFFFFFFull};
+#endif
+ return (__m128i)vec_and(_mm_loadu_si128(a), (vector unsigned char)mask);
+}
+
+Y_FORCE_INLINE void _mm_storel_epi64(__m128i* a, __m128i b) {
+ *(long long*)a = ((__v2di)b)[0];
+}
+
+Y_FORCE_INLINE double _mm_cvtsd_f64(__m128d a) {
+ return ((__v2df)a)[0];
+}
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wuninitialized"
+Y_FORCE_INLINE __m128d _mm_undefined_pd(void) {
+ __m128d ans = ans;
+ return ans;
+}
+#pragma GCC diagnostic pop
+
+Y_FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double* b) {
+ __v2df result = (__v2df)a;
+ result[1] = *b;
+ return (__m128d)result;
+}
+
+Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) {
+ __v2df result = (__v2df)a;
+ result[0] = *b;
+ return (__m128d)result;
+}
+
+Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i a) {
+ return (__m128)a;
+}
+
+Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 a) {
+ return (__m128i)a;
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpgt_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_cmpgt((__v16qi)a, (__v16qi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpgt_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_cmpgt((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpgt_epi32(__m128i a, __m128i b) {
+ return (__m128i)vec_cmpgt((__v4si)a, (__v4si)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmpgt_epi64(__m128i a, __m128i b) {
+ return vec_cmpgt((vector signed long long)a, (vector signed long long)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmplt_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_cmplt((__v16qi)a, (__v16qi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmplt_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_cmplt((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmplt_epi32(__m128i a, __m128i b) {
+ return (__m128i)vec_cmplt((__v4si)a, (__v4si)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_cmplt_epi64(__m128i a, __m128i b) {
+ return vec_cmplt((vector signed long long)a, (vector signed long long)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_sad_epu8(__m128i A, __m128i B) {
+ __v16qu a, b;
+ __v16qu vmin, vmax, vabsdiff;
+ __v4si vsum;
+ const __v4su zero = {0, 0, 0, 0};
+ __v4si result;
+
+ a = (__v16qu)A;
+ b = (__v16qu)B;
+ vmin = vec_min(a, b);
+ vmax = vec_max(a, b);
+ vabsdiff = vec_sub(vmax, vmin);
+ /* Sum four groups of bytes into integers. */
+ vsum = (__vector signed int)vec_sum4s(vabsdiff, zero);
+ /* Sum across four integers with two integer results. */
+ result = vec_sum2s(vsum, (__vector signed int)zero);
+ /* Rotate the sums into the correct position. */
+#ifdef __LITTLE_ENDIAN__
+ result = vec_sld(result, result, 4);
+#elif __BIG_ENDIAN__
+ result = vec_sld(result, result, 6);
+#endif
+ /* Rotate the sums into the correct position. */
+ return (__m128i)result;
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_subs((__v16qi)a, (__v16qi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_subs((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) {
+ return (__m128i)vec_subs((__v16qu)a, (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) {
+ return (__m128i)vec_subs((__v8hu)a, (__v8hu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_adds_epi8(__m128i a, __m128i b) {
+ return (__m128i)vec_adds((__v16qi)a, (__v16qi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_adds_epi16(__m128i a, __m128i b) {
+ return (__m128i)vec_adds((__v8hi)a, (__v8hi)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_adds_epu8(__m128i a, __m128i b) {
+ return (__m128i)vec_adds((__v16qu)a, (__v16qu)b);
+}
+
+Y_FORCE_INLINE __m128i _mm_adds_epu16(__m128i a, __m128i b) {
+ return (__m128i)vec_adds((__v8hu)a, (__v8hu)b);
+}
+
+Y_FORCE_INLINE __m128d _mm_castsi128_pd(__m128i a) {
+ return (__m128d)a;
+}
+
+Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) {
+ __builtin_prefetch(p);
+}
+
+Y_FORCE_INLINE __m128i _mm_hadd_epi16(__m128i a, __m128i b) {
+ const __v16qu p = { 0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29 };
+ const __v16qu q = { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 };
+ __v8hi c = vec_perm((__v8hi)a, (__v8hi)b, p);
+ __v8hi d = vec_perm((__v8hi)a, (__v8hi)b, q);
+ return (__m128i)vec_add(c, d);
+}
diff --git a/library/cpp/sse/sse.cpp b/library/cpp/sse/sse.cpp
new file mode 100644
index 0000000000..d0d1de9930
--- /dev/null
+++ b/library/cpp/sse/sse.cpp
@@ -0,0 +1 @@
+#include "sse.h"
diff --git a/library/cpp/sse/sse.h b/library/cpp/sse/sse.h
new file mode 100644
index 0000000000..19bac17de0
--- /dev/null
+++ b/library/cpp/sse/sse.h
@@ -0,0 +1,33 @@
+#pragma once
+
+/*
+ The header chooses appropriate SSE support.
+ On Intel: SSE intrinsics
+ On ARM64: translation to NEON intrinsics or software emulation
+ On PowerPc: translation to Altivec intrinsics or software emulation
+*/
+/* Author: Vitaliy Manushkin <agri@yandex-team.ru>, Danila Kutenin <danlark@yandex-team.ru> */
+
+#include <util/system/platform.h>
+
+#if (defined(_i386_) || defined(_x86_64_)) && defined(_sse_)
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <pmmintrin.h>
+#define ARCADIA_SSE
+#if defined(_ssse3_)
+#include <tmmintrin.h>
+#endif
+#if defined(_sse4_1_)
+#include <smmintrin.h>
+#endif
+#if defined(_sse4_2_)
+#include <nmmintrin.h>
+#endif
+#elif defined(_arm64_)
+#include "sse2neon.h"
+#define ARCADIA_SSE
+#elif defined(_ppc64_)
+#include "powerpc.h"
+#define ARCADIA_SSE
+#endif
diff --git a/library/cpp/sse/sse2neon.h b/library/cpp/sse/sse2neon.h
new file mode 100644
index 0000000000..695dbd3041
--- /dev/null
+++ b/library/cpp/sse/sse2neon.h
@@ -0,0 +1,1045 @@
+#pragma once
+
+/*
+ The header contains inlining code
+ which translates SSE intrinsics to NEON intrinsics or software emulation.
+ You are encouraged for commitments.
+ Add missing intrinsics, add unittests, purify the implementation,
+ merge and simplify templates.
+ Warning: The code is made in deep nights, so it surely contains bugs,
+ imperfections, flaws and all other kinds of errors and mistakes.
+*/
+/* Author: Vitaliy Manushkin <agri@yandex-team.ru> */
+
+#include <util/system/platform.h>
+#include <util/system/compiler.h>
+#include <util/system/types.h>
+
+#if !defined(_arm64_)
+#error "This header is for ARM64 (aarch64) platform only. " \
+ "Include sse.h instead of including this header directly."
+#endif
+
+#include <arm_neon.h>
+
+union __m128i {
+ uint64x2_t AsUi64x2;
+ int64x2_t AsSi64x2;
+
+ uint32x4_t AsUi32x4;
+ int32x4_t AsSi32x4;
+
+ uint16x8_t AsUi16x8;
+ int16x8_t AsSi16x8;
+
+ uint8x16_t AsUi8x16;
+ int8x16_t AsSi8x16;
+
+ float32x4_t AsFloat32x4;
+ float64x2_t AsFloat64x2;
+};
+
+union __m128 {
+ float32x4_t AsFloat32x4;
+ float64x2_t AsFloat64x2;
+
+ uint32x4_t AsUi32x4;
+ int32x4_t AsSi32x4;
+
+ uint64x2_t AsUi64x2;
+ int64x2_t AsSi64x2;
+
+ uint8x16_t AsUi8x16;
+ int8x16_t AsSi8x16;
+
+ __m128i As128i;
+};
+
+typedef float64x2_t __m128d;
+
+enum _mm_hint
+{
+ /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */
+ _MM_HINT_ET0 = 7,
+ _MM_HINT_ET1 = 6,
+ _MM_HINT_T0 = 3,
+ _MM_HINT_T1 = 2,
+ _MM_HINT_T2 = 1,
+ _MM_HINT_NTA = 0
+};
+
+Y_FORCE_INLINE void _mm_prefetch(const void *p, enum _mm_hint) {
+ __builtin_prefetch(p);
+}
+
+template <typename TType>
+struct TQType;
+
+template <>
+struct TQType<uint8x16_t> {
+ static inline uint8x16_t& As(__m128i& value) {
+ return value.AsUi8x16;
+ }
+ static inline const uint8x16_t& As(const __m128i& value) {
+ return value.AsUi8x16;
+ }
+};
+
+template <>
+struct TQType<int8x16_t> {
+ static inline int8x16_t& As(__m128i& value) {
+ return value.AsSi8x16;
+ }
+ static inline const int8x16_t& As(const __m128i& value) {
+ return value.AsSi8x16;
+ }
+};
+
+template <>
+struct TQType<uint16x8_t> {
+ static inline uint16x8_t& As(__m128i& value) {
+ return value.AsUi16x8;
+ }
+ static inline const uint16x8_t& As(const __m128i& value) {
+ return value.AsUi16x8;
+ }
+};
+
+template <>
+struct TQType<int16x8_t> {
+ static inline int16x8_t& As(__m128i& value) {
+ return value.AsSi16x8;
+ }
+ static inline const int16x8_t& As(const __m128i& value) {
+ return value.AsSi16x8;
+ }
+};
+
+template <>
+struct TQType<uint32x4_t> {
+ static inline uint32x4_t& As(__m128i& value) {
+ return value.AsUi32x4;
+ }
+ static inline const uint32x4_t& As(const __m128i& value) {
+ return value.AsUi32x4;
+ }
+};
+
+template <>
+struct TQType<int32x4_t> {
+ static inline int32x4_t& As(__m128i& value) {
+ return value.AsSi32x4;
+ }
+ static inline const int32x4_t& As(const __m128i& value) {
+ return value.AsSi32x4;
+ }
+};
+
+template <>
+struct TQType<uint64x2_t> {
+ static inline uint64x2_t& As(__m128i& value) {
+ return value.AsUi64x2;
+ }
+ static inline const uint64x2_t& As(const __m128i& value) {
+ return value.AsUi64x2;
+ }
+ static inline uint64x2_t& As(__m128& value) {
+ return value.AsUi64x2;
+ }
+ static inline const uint64x2_t& As(const __m128& value) {
+ return value.AsUi64x2;
+ }
+};
+
+template <>
+struct TQType<int64x2_t> {
+ static inline int64x2_t& As(__m128i& value) {
+ return value.AsSi64x2;
+ }
+ static inline const int64x2_t& As(const __m128i& value) {
+ return value.AsSi64x2;
+ }
+};
+
+template <typename TValue>
+struct TBaseWrapper {
+ TValue Value;
+
+ Y_FORCE_INLINE
+ operator TValue&() {
+ return Value;
+ }
+
+ Y_FORCE_INLINE
+ operator const TValue&() const {
+ return Value;
+ }
+};
+
+template <typename TOp, typename TFunc, TFunc* func,
+ typename TDup, TDup* dupfunc>
+struct TWrapperSingleDup: public TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TWrapperSingleDup(const __m128i& op, const int shift) {
+ TQType<TOp>::As(Value) = func(TQType<TOp>::As(op), dupfunc(shift));
+ }
+};
+
+template <typename TOp, typename TFunc, TFunc* func,
+ typename TDup, TDup* dupfunc>
+struct TWrapperSingleNegDup: public TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TWrapperSingleNegDup(const __m128i& op, const int shift) {
+ TQType<TOp>::As(Value) = func(TQType<TOp>::As(op), dupfunc(-shift));
+ }
+};
+
+inline __m128i _mm_srl_epi16(__m128i a, __m128i count) {
+ __m128i res;
+ res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(-count.AsUi16x8[0]));
+ return res;
+}
+
+
+inline __m128i _mm_srl_epi32(__m128i a, __m128i count) {
+ __m128i res;
+ res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(-count.AsUi32x4[0]));
+ return res;
+}
+
+inline __m128i _mm_srl_epi64(__m128i a, __m128i count) {
+ __m128i res;
+ res.AsUi64x2 = vshlq_u64(a.AsUi64x2, vdupq_n_s64(-count.AsUi64x2[0]));
+ return res;
+}
+
+inline __m128i _mm_srai_epi16(__m128i a, int count) {
+ __m128i res;
+ res.AsSi16x8 = vqshlq_s16(a.AsSi16x8, vdupq_n_s16(-count));
+ return res;
+}
+
+inline __m128i _mm_srai_epi32(__m128i a, int count) {
+ __m128i res;
+ res.AsSi32x4 = vqshlq_s32(a.AsSi32x4, vdupq_n_s32(-count));
+ return res;
+}
+
+using _mm_srli_epi16 =
+ TWrapperSingleNegDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16,
+ decltype(vdupq_n_s16), vdupq_n_s16>;
+using _mm_srli_epi32 =
+ TWrapperSingleNegDup<uint32x4_t, decltype(vshlq_u32), vshlq_u32,
+ decltype(vdupq_n_s32), vdupq_n_s32>;
+using _mm_srli_epi64 =
+ TWrapperSingleNegDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64,
+ decltype(vdupq_n_s64), vdupq_n_s64>;
+
+
+inline __m128i _mm_sll_epi16(__m128i a, __m128i count) {
+ __m128i res;
+ res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(count.AsUi16x8[0]));
+ return res;
+}
+
+
+inline __m128i _mm_sll_epi32(__m128i a, __m128i count) {
+ __m128i res;
+ res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(count.AsUi32x4[0]));
+ return res;
+}
+
+inline __m128i _mm_sll_epi64(__m128i a, __m128i count) {
+ __m128i res;
+ res.AsUi64x2 = vshlq_u64(a.AsUi64x2, vdupq_n_s64(count.AsUi64x2[0]));
+ return res;
+}
+
+using _mm_slli_epi16 =
+ TWrapperSingleDup<uint16x8_t, decltype(vshlq_u16), vshlq_u16,
+ decltype(vdupq_n_s16), vdupq_n_s16>;
+using _mm_slli_epi32 =
+ TWrapperSingleDup<uint32x4_t, decltype(vshlq_u32), vshlq_u32,
+ decltype(vdupq_n_s32), vdupq_n_s32>;
+using _mm_slli_epi64 =
+ TWrapperSingleDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64,
+ decltype(vdupq_n_s64), vdupq_n_s64>;
+
+template <typename TOp, typename TFunc, TFunc* func, typename... TParams>
+struct TWrapperDual : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TWrapperDual(const __m128i& op1, const __m128i& op2, TParams... params) {
+ TQType<TOp>::As(Value) = (TOp)
+ func(TQType<TOp>::As(op1),
+ TQType<TOp>::As(op2),
+ params...);
+ }
+};
+
+template <typename TOp, typename TFunc, TFunc* func, typename... TParams>
+struct TWrapperDualSwap : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TWrapperDualSwap(const __m128i& op1, const __m128i& op2, TParams... params) {
+ TQType<TOp>::As(Value) =
+ func(TQType<TOp>::As(op2),
+ TQType<TOp>::As(op1),
+ params...);
+ }
+};
+
+template <typename TOp, typename TFunc, TFunc* func, typename TArgument = __m128>
+struct TWrapperDualF : TBaseWrapper<TArgument> {
+ Y_FORCE_INLINE
+ TWrapperDualF(const TArgument& op1, const TArgument& op2) {
+ TQType<TOp>::As(TBaseWrapper<TArgument>::Value) = (TOp) func(TQType<TOp>::As(op1), TQType<TOp>::As(op2));
+ }
+};
+
+using _mm_or_si128 = TWrapperDual<uint64x2_t, decltype(vorrq_u64), vorrq_u64>;
+using _mm_and_si128 = TWrapperDual<uint64x2_t, decltype(vandq_u64), vandq_u64>;
+using _mm_andnot_si128 =
+ TWrapperDualSwap<uint64x2_t, decltype(vbicq_u64), vbicq_u64>;
+using _mm_xor_si128 = TWrapperDual<uint64x2_t, decltype(veorq_u64), veorq_u64>;
+
+using _mm_add_epi8 = TWrapperDual<uint8x16_t, decltype(vaddq_u8), vaddq_u8>;
+using _mm_add_epi16 = TWrapperDual<uint16x8_t, decltype(vaddq_u16), vaddq_u16>;
+using _mm_add_epi32 = TWrapperDual<uint32x4_t, decltype(vaddq_u32), vaddq_u32>;
+using _mm_add_epi64 = TWrapperDual<uint64x2_t, decltype(vaddq_u64), vaddq_u64>;
+
+inline __m128i _mm_madd_epi16(__m128i a, __m128i b) {
+ int32x4_t aLow;
+ int32x4_t aHigh;
+ int32x4_t bLow;
+ int32x4_t bHigh;
+ #ifdef __LITTLE_ENDIAN__
+ aLow[0] = a.AsSi16x8[0]; //!< I couldn't find vector instructions to do that. Feel free to fix this code.
+ aLow[1] = a.AsSi16x8[2];
+ aLow[2] = a.AsSi16x8[4];
+ aLow[3] = a.AsSi16x8[6];
+
+ aHigh[0] = a.AsSi16x8[1];
+ aHigh[1] = a.AsSi16x8[3];
+ aHigh[2] = a.AsSi16x8[5];
+ aHigh[3] = a.AsSi16x8[7];
+
+ bLow[0] = b.AsSi16x8[0];
+ bLow[1] = b.AsSi16x8[2];
+ bLow[2] = b.AsSi16x8[4];
+ bLow[3] = b.AsSi16x8[6];
+
+ bHigh[0] = b.AsSi16x8[1];
+ bHigh[1] = b.AsSi16x8[3];
+ bHigh[2] = b.AsSi16x8[5];
+ bHigh[3] = b.AsSi16x8[7];
+ #else
+ #error Not implemented yet. Do it yourself.
+ #endif
+
+ const int32x4_t lowMul = vmulq_u32(aLow, bLow);
+ const int32x4_t highMul = vmulq_u32(aHigh, bHigh);
+ __m128i res;
+ res.AsSi32x4 = vaddq_u32(lowMul, highMul);
+ return res;
+}
+
+using _mm_sub_epi8 = TWrapperDual<uint8x16_t, decltype(vsubq_u8), vsubq_u8>;
+using _mm_sub_epi16 = TWrapperDual<uint16x8_t, decltype(vsubq_u16), vsubq_u16>;
+using _mm_sub_epi32 = TWrapperDual<uint32x4_t, decltype(vsubq_u32), vsubq_u32>;
+using _mm_sub_epi64 = TWrapperDual<uint64x2_t, decltype(vsubq_u64), vsubq_u64>;
+
+using _mm_unpacklo_epi8 =
+ TWrapperDual<uint8x16_t, decltype(vzip1q_u8), vzip1q_u8>;
+using _mm_unpackhi_epi8 =
+ TWrapperDual<uint8x16_t, decltype(vzip2q_u8), vzip2q_u8>;
+using _mm_unpacklo_epi16 =
+ TWrapperDual<uint16x8_t, decltype(vzip1q_u16), vzip1q_u16>;
+using _mm_unpackhi_epi16 =
+ TWrapperDual<uint16x8_t, decltype(vzip2q_u16), vzip2q_u16>;
+using _mm_unpacklo_epi32 =
+ TWrapperDual<uint32x4_t, decltype(vzip1q_u32), vzip1q_u32>;
+using _mm_unpackhi_epi32 =
+ TWrapperDual<uint32x4_t, decltype(vzip2q_u32), vzip2q_u32>;
+using _mm_unpacklo_epi64 =
+ TWrapperDual<uint64x2_t, decltype(vzip1q_u64), vzip1q_u64>;
+using _mm_unpackhi_epi64 =
+ TWrapperDual<uint64x2_t, decltype(vzip2q_u64), vzip2q_u64>;
+
+using _mm_cmpeq_epi8 =
+ TWrapperDual<uint8x16_t, decltype(vceqq_u8), vceqq_u8>;
+using _mm_cmpeq_epi16 =
+ TWrapperDual<uint16x8_t, decltype(vceqq_u16), vceqq_u16>;
+using _mm_cmpeq_epi32 =
+ TWrapperDual<uint32x4_t, decltype(vceqq_u32), vceqq_u32>;
+
+using _mm_cmpgt_epi8 =
+ TWrapperDual<int8x16_t, decltype(vcgtq_s8), vcgtq_s8>;
+using _mm_cmpgt_epi16 =
+ TWrapperDual<int16x8_t, decltype(vcgtq_s16), vcgtq_s16>;
+using _mm_cmpgt_epi32 =
+ TWrapperDual<int32x4_t, decltype(vcgtq_s32), vcgtq_s32>;
+
+using _mm_cmplt_epi8 =
+ TWrapperDual<int8x16_t, decltype(vcltq_s8), vcltq_s8>;
+using _mm_cmplt_epi16 =
+ TWrapperDual<int16x8_t, decltype(vcltq_s16), vcltq_s16>;
+using _mm_cmplt_epi32 =
+ TWrapperDual<int32x4_t, decltype(vcltq_s32), vcltq_s32>;
+
+Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* ptr) {
+ __m128i result;
+ result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr);
+ return result;
+}
+
+Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* ptr) {
+ __m128i result;
+ result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr);
+ return result;
+}
+
+Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* ptr) {
+ return _mm_loadu_si128(ptr);
+}
+
+Y_FORCE_INLINE void _mm_storeu_si128(__m128i* ptr, const __m128i& op) {
+ vst1q_u64((uint64_t*)ptr, op.AsUi64x2);
+}
+
+Y_FORCE_INLINE void
+_mm_store_si128(__m128i* ptr, const __m128i& op) {
+ vst1q_u64((uint64_t*)ptr, op.AsUi64x2);
+}
+
+template <typename TOp, typename TFunc, TFunc* func, typename... TParams>
+struct TWrapperSimple : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TWrapperSimple(TParams... params) {
+ TQType<TOp>::As(Value) = func(params...);
+ }
+};
+
+template <typename TOp, typename TFunc, TFunc* func, typename... TParams>
+struct TWrapperSimpleF : TBaseWrapper<__m128> {
+ Y_FORCE_INLINE
+ TWrapperSimpleF(TParams... params) {
+ TQType<TOp>::As(Value) = func(params...);
+ }
+};
+
+using _mm_set1_epi8 =
+ TWrapperSimple<int8x16_t, decltype(vdupq_n_s8), vdupq_n_s8, const char>;
+using _mm_set1_epi16 =
+ TWrapperSimple<int16x8_t, decltype(vdupq_n_s16), vdupq_n_s16, const ui16>;
+using _mm_set1_epi32 =
+ TWrapperSimple<int32x4_t, decltype(vdupq_n_s32), vdupq_n_s32, const ui32>;
+
+struct _mm_setzero_si128 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_setzero_si128() {
+ TQType<uint64x2_t>::As(Value) = vdupq_n_u64(0);
+ }
+};
+
+struct _mm_loadl_epi64 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_loadl_epi64(const __m128i* p) {
+ uint64x1_t im = vld1_u64((const uint64_t*)p);
+ TQType<uint64x2_t>::As(Value) = vcombine_u64(im, vdup_n_u64(0));
+ }
+};
+
+struct _mm_storel_epi64 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_storel_epi64(__m128i* a, __m128i op) {
+ vst1_u64((uint64_t*)a, vget_low_u64(op.AsUi64x2));
+ }
+};
+
+struct ShuffleStruct4 {
+ ui8 x[4];
+};
+
+Y_FORCE_INLINE ShuffleStruct4
+_MM_SHUFFLE(ui8 x4, ui8 x3, ui8 x2, ui8 x1) {
+ ShuffleStruct4 result;
+ result.x[0] = x1;
+ result.x[1] = x2;
+ result.x[2] = x3;
+ result.x[3] = x4;
+ return result;
+}
+
+Y_FORCE_INLINE __m128i
+_mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) {
+ __m128i result;
+ const ui8 xi[4] = {
+ ui8(op2.x[0] * 4), ui8(op2.x[1] * 4),
+ ui8(op2.x[2] * 4), ui8(op2.x[3] * 4)
+ };
+ const uint8x16_t transform = {
+ ui8(xi[0]), ui8(xi[0] + 1), ui8(xi[0] + 2), ui8(xi[0] + 3),
+ ui8(xi[1]), ui8(xi[1] + 1), ui8(xi[1] + 2), ui8(xi[1] + 3),
+ ui8(xi[2]), ui8(xi[2] + 1), ui8(xi[2] + 2), ui8(xi[2] + 3),
+ ui8(xi[3]), ui8(xi[3] + 1), ui8(xi[3] + 2), ui8(xi[3] + 3)
+ };
+ result.AsUi8x16 = vqtbl1q_u8(op1.AsUi8x16, transform);
+ return result;
+}
+
+Y_FORCE_INLINE int
+_mm_movemask_epi8(const __m128i& op) {
+ uint8x16_t mask = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80};
+ uint8x16_t opmasked = vandq_u8(op.AsUi8x16, mask);
+ int8x16_t byteshifter = {
+ 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7, 0, -7};
+ uint8x16_t opshifted = vshlq_u8(opmasked, byteshifter);
+ int16x8_t wordshifter = {-7, -5, -3, -1, 1, 3, 5, 7};
+ uint16x8_t wordshifted =
+ vshlq_u16(vreinterpretq_u16_u8(opshifted), wordshifter);
+ return vaddvq_u16(wordshifted);
+}
+
+template <int imm>
+struct THelper_mm_srli_si128 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ THelper_mm_srli_si128(const __m128i a) {
+ const auto zero = vdupq_n_u8(0);
+ TQType<uint8x16_t>::As(Value) = vextq_u8(a.AsUi8x16, zero, imm);
+ }
+};
+
+template <>
+struct THelper_mm_srli_si128<16> : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ THelper_mm_srli_si128(const __m128i /* a */) {
+ const auto zero = vdupq_n_u8(0);
+ TQType<uint8x16_t>::As(Value) = zero;
+ }
+};
+
+#define _mm_srli_si128(a, imm) THelper_mm_srli_si128<imm>(a)
+
+template<int imm>
+inline uint8x16_t vextq_u8_function(uint8x16_t a, uint8x16_t b) {
+ return vextq_u8(a, b, imm);
+}
+
+template<>
+inline uint8x16_t vextq_u8_function<16>(uint8x16_t /* a */, uint8x16_t b) {
+ return b;
+}
+
+
+template <int imm>
+struct THelper_mm_slli_si128 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ THelper_mm_slli_si128(const __m128i a) {
+ auto zero = vdupq_n_u8(0);
+ TQType<uint8x16_t>::As(Value) = vextq_u8_function<16 - imm>(zero, a.AsUi8x16);
+ }
+};
+
+#define _mm_slli_si128(a, imm) THelper_mm_slli_si128<imm>(a)
+
+Y_FORCE_INLINE int _mm_cvtsi128_si32(const __m128i& op) {
+ return vgetq_lane_s32(op.AsSi32x4, 0);
+}
+
+struct _mm_set_epi16 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_set_epi16(const short w7, const short w6,
+ const short w5, const short w4,
+ const short w3, const short w2,
+ const short w1, const short w0) {
+ int16x4_t d0 = {w0, w1, w2, w3};
+ int16x4_t d1 = {w4, w5, w6, w7};
+ TQType<int16x8_t>::As(Value) = vcombine_s16(d0, d1);
+ }
+};
+
+struct _mm_setr_epi16 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_setr_epi16(const short w7, const short w6,
+ const short w5, const short w4,
+ const short w3, const short w2,
+ const short w1, const short w0) {
+ int16x4_t d0 = {w7, w6, w5, w4};
+ int16x4_t d1 = {w3, w2, w1, w0};
+ TQType<int16x8_t>::As(Value) = vcombine_s16(d0, d1);
+ }
+};
+
+struct _mm_set_epi32 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_set_epi32(const int x3, const int x2,
+ const int x1, const int x0) {
+ int32x2_t d0 = {x0, x1};
+ int32x2_t d1 = {x2, x3};
+ TQType<int32x4_t>::As(Value) = vcombine_s32(d0, d1);
+ }
+};
+
+struct _mm_setr_epi32 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_setr_epi32(const int x3, const int x2,
+ const int x1, const int x0) {
+ int32x2_t d0 = {x3, x2};
+ int32x2_t d1 = {x1, x0};
+ TQType<int32x4_t>::As(Value) = vcombine_s32(d0, d1);
+ }
+};
+
+struct _mm_cvtsi32_si128 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_cvtsi32_si128(int op) {
+ auto zero = vdupq_n_s32(0);
+ TQType<int32x4_t>::As(Value) = vsetq_lane_s32(op, zero, 0);
+ }
+};
+
+struct _mm_cvtsi64_si128 : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ _mm_cvtsi64_si128(i64 op) {
+ auto zero = vdupq_n_s64(0);
+ TQType<int64x2_t>::As(Value) = vsetq_lane_s64(op, zero, 0);
+ }
+};
+
+template <typename TOpOut, typename TOpIn,
+ typename TFunc, TFunc* func,
+ typename TCombine, TCombine* combine>
+struct TCombineWrapper : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TCombineWrapper(const __m128i op1, const __m128i op2) {
+ TQType<TOpOut>::As(Value) =
+ combine(func(TQType<TOpIn>::As(op1)),
+ func(TQType<TOpIn>::As(op2)));
+ }
+};
+
+using _mm_packs_epi16 =
+ TCombineWrapper<int8x16_t, int16x8_t,
+ decltype(vqmovn_s16), vqmovn_s16,
+ decltype(vcombine_s8), vcombine_s8>;
+using _mm_packs_epi32 =
+ TCombineWrapper<int16x8_t, int32x4_t,
+ decltype(vqmovn_s32), vqmovn_s32,
+ decltype(vcombine_s16), vcombine_s16>;
+using _mm_packus_epi16 =
+ TCombineWrapper<uint8x16_t, int16x8_t,
+ decltype(vqmovun_s16), vqmovun_s16,
+ decltype(vcombine_u8), vcombine_u8>;
+
+template <typename TOpOut, typename TOpIn,
+ typename TFunc, TFunc* func, typename... TParams>
+struct TScalarOutWrapper : TBaseWrapper<TOpOut> {
+ Y_FORCE_INLINE
+ TScalarOutWrapper(const __m128i op, TParams... params) {
+ TBaseWrapper<TOpOut>::Value =
+ func(TQType<TOpIn>::As(op), params...);
+ }
+};
+
+template<int imm>
+int extract_epi8_arm(__m128i arg) {
+ return vgetq_lane_u8(arg.AsUi8x16, imm);
+}
+
+template<int imm>
+int extract_epi16_arm(__m128i arg) {
+ return vgetq_lane_u16(arg.AsUi16x8, imm);
+}
+
+template<int imm>
+int extract_epi32_arm(__m128i arg) {
+ return vgetq_lane_s32(arg.AsSi32x4, imm);
+}
+
+template<int imm>
+long long extract_epi64_arm(__m128i arg) {
+ return vgetq_lane_s64(arg.AsSi64x2, imm);
+}
+
+#define _mm_extract_epi8(op, imm) extract_epi8_arm<imm>(op)
+#define _mm_extract_epi16(op, imm) extract_epi16_arm<imm>(op)
+#define _mm_extract_epi32(op, imm) extract_epi32_arm<imm>(op)
+#define _mm_extract_epi64(op, imm) extract_epi64_arm<imm>(op)
+#define _mm_extract_ps(op, imm) _mm_extract_epi32(op, imm)
+
+static Y_FORCE_INLINE
+__m128i _mm_mul_epu32(__m128i op1, __m128i op2) {
+ __m128i result;
+ uint32x4_t r1 = vuzp1q_u32(op1.AsUi32x4, op2.AsUi32x4);
+ uint32x4_t r2 = vuzp1q_u32(op2.AsUi32x4, op1.AsUi32x4);
+ result.AsUi64x2 = vmull_u32(vget_low_u32(r1), vget_low_u32(r2));
+ return result;
+}
+
+template <>
+struct TQType<float32x4_t> {
+ static inline float32x4_t& As(__m128& value) {
+ return value.AsFloat32x4;
+ }
+
+ static inline const float32x4_t& As(const __m128& value) {
+ return value.AsFloat32x4;
+ }
+
+ static inline float32x4_t& As(__m128i& value) {
+ return value.AsFloat32x4;
+ }
+
+ static inline const float32x4_t& As(const __m128i& value) {
+ return value.AsFloat32x4;
+ }
+};
+
+template <>
+struct TQType<float64x2_t> {
+ static inline float64x2_t& As(__m128& value) {
+ return value.AsFloat64x2;
+ }
+
+ static inline const float64x2_t& As(const __m128& value) {
+ return value.AsFloat64x2;
+ }
+
+ static inline float64x2_t& As(__m128i& value) {
+ return value.AsFloat64x2;
+ }
+
+ static inline const float64x2_t& As(const __m128i& value) {
+ return value.AsFloat64x2;
+ }
+
+ static inline float64x2_t& As(__m128d& value) {
+ return value;
+ }
+
+ static inline const float64x2_t& As(const __m128d& value) {
+ return value;
+ }
+};
+
+using _mm_set1_ps = TWrapperSimpleF<float32x4_t,
+ decltype(vdupq_n_f32), vdupq_n_f32, const float>;
+using _mm_set_ps1 = TWrapperSimpleF<float32x4_t,
+ decltype(vdupq_n_f32), vdupq_n_f32, const float>;
+
+struct _mm_setzero_ps : TBaseWrapper<__m128> {
+ Y_FORCE_INLINE
+ _mm_setzero_ps() {
+ TQType<float32x4_t>::As(Value) = vdupq_n_f32(0.);
+ }
+};
+
+Y_FORCE_INLINE __m128d _mm_setzero_pd() {
+ return vdupq_n_f64(0.);
+}
+
+Y_FORCE_INLINE __m128 _mm_loadu_ps(const float* ptr) {
+ __m128 result;
+ result.AsFloat32x4 = vld1q_f32(ptr);
+ return result;
+}
+
+Y_FORCE_INLINE __m128 _mm_load_ps(const float* ptr) {
+ __m128 result;
+ result.AsFloat32x4 = vld1q_f32(ptr);
+ return result;
+}
+
+Y_FORCE_INLINE void _mm_storeu_ps(float* ptr, const __m128& op) {
+ vst1q_f32(ptr, op.AsFloat32x4);
+}
+
+Y_FORCE_INLINE void _mm_store_ps(float* ptr, const __m128& op) {
+ vst1q_f32(ptr, op.AsFloat32x4);
+}
+
+struct _mm_set_ps : TBaseWrapper<__m128> {
+ Y_FORCE_INLINE
+ _mm_set_ps(const float x3, const float x2,
+ const float x1, const float x0) {
+ float32x2_t d0 = {x0, x1};
+ float32x2_t d1 = {x2, x3};
+ TQType<float32x4_t>::As(Value) = vcombine_f32(d0, d1);
+ }
+};
+
+Y_FORCE_INLINE __m128d _mm_set_pd(double d1, double d0) {
+ const float64x1_t p0 = {d0};
+ const float64x1_t p1 = {d1};
+ return vcombine_f64(p0, p1);
+}
+
+Y_FORCE_INLINE __m128d _mm_loadu_pd(const double* d) {
+ __m128d res;
+ res = vld1q_f64(d);
+ return res;
+}
+
+Y_FORCE_INLINE void _mm_storeu_pd(double* res, __m128d a) {
+ vst1q_f64(res, a);
+}
+
+Y_FORCE_INLINE void _mm_store_pd(double* res, __m128d a) {
+ vst1q_f64(res, a);
+}
+
+using _mm_add_ps = TWrapperDualF<float32x4_t, decltype(vaddq_f32), vaddq_f32>;
+using _mm_sub_ps = TWrapperDualF<float32x4_t, decltype(vsubq_f32), vsubq_f32>;
+using _mm_mul_ps = TWrapperDualF<float32x4_t, decltype(vmulq_f32), vmulq_f32>;
+using _mm_div_ps = TWrapperDualF<float32x4_t, decltype(vdivq_f32), vdivq_f32>;
+using _mm_cmpeq_ps = TWrapperDualF<float32x4_t, decltype(vceqq_f32), vceqq_f32>;
+using _mm_cmpgt_ps = TWrapperDualF<float32x4_t, decltype(vcgtq_f32), vcgtq_f32>;
+using _mm_max_ps = TWrapperDualF<float32x4_t, decltype(vmaxq_f32), vmaxq_f32>;
+using _mm_min_ps = TWrapperDualF<float32x4_t, decltype(vminq_f32), vminq_f32>;
+
+using _mm_add_pd = TWrapperDualF<float64x2_t, decltype(vaddq_f64), vaddq_f64, __m128d>;
+using _mm_sub_pd = TWrapperDualF<float64x2_t, decltype(vsubq_f64), vsubq_f64, __m128d>;
+using _mm_mul_pd = TWrapperDualF<float64x2_t, decltype(vmulq_f64), vmulq_f64, __m128d>;
+using _mm_div_pd = TWrapperDualF<float64x2_t, decltype(vdivq_f64), vdivq_f64, __m128d>;
+
+struct _mm_and_ps : TBaseWrapper<__m128> {
+ Y_FORCE_INLINE
+ _mm_and_ps(const __m128& op1, const __m128& op2) {
+ TQType<uint64x2_t>::As(Value) =
+ vandq_u64(TQType<uint64x2_t>::As(op1),
+ TQType<uint64x2_t>::As(op2));
+ }
+};
+
+Y_FORCE_INLINE __m128d _mm_and_pd(__m128d a, __m128d b) {
+ return vandq_u64(a, b);
+}
+
+Y_FORCE_INLINE void _MM_TRANSPOSE4_PS(__m128& op0, __m128& op1, __m128& op2, __m128& op3) {
+ float64x2_t im0 =
+ (float64x2_t)vtrn1q_f32(op0.AsFloat32x4, op1.AsFloat32x4);
+ float64x2_t im1 =
+ (float64x2_t)vtrn2q_f32(op0.AsFloat32x4, op1.AsFloat32x4);
+ float64x2_t im2 =
+ (float64x2_t)vtrn1q_f32(op2.AsFloat32x4, op3.AsFloat32x4);
+ float64x2_t im3 =
+ (float64x2_t)vtrn2q_f32(op2.AsFloat32x4, op3.AsFloat32x4);
+
+ TQType<float64x2_t>::As(op0) = vtrn1q_f64(im0, im2);
+ TQType<float64x2_t>::As(op1) = vtrn1q_f64(im1, im3);
+ TQType<float64x2_t>::As(op2) = vtrn2q_f64(im0, im2);
+ TQType<float64x2_t>::As(op3) = vtrn2q_f64(im1, im3);
+};
+
+Y_FORCE_INLINE __m128 _mm_castsi128_ps(__m128i op) {
+ return reinterpret_cast<__m128&>(op);
+}
+
+Y_FORCE_INLINE __m128i _mm_castps_si128(__m128 op) {
+ return reinterpret_cast<__m128i&>(op);
+}
+
+template <typename TOpOut, typename TOpIn,
+ typename TFunc, TFunc* func, typename... TParams>
+struct TCvtS2FWrapperSingle : TBaseWrapper<__m128> {
+ Y_FORCE_INLINE
+ TCvtS2FWrapperSingle(const __m128i& op, TParams... params) {
+ TQType<TOpOut>::As(Value) =
+ func(TQType<TOpIn>::As(op), params...);
+ }
+};
+
+using _mm_cvtepi32_ps =
+ TCvtS2FWrapperSingle<float32x4_t, int32x4_t,
+ decltype(vcvtq_f32_s32), vcvtq_f32_s32>;
+
+template <typename TOpOut, typename TOpIn,
+ typename TFunc, TFunc* func, typename... TParams>
+struct TCvtF2SWrapperSingle : TBaseWrapper<__m128i> {
+ Y_FORCE_INLINE
+ TCvtF2SWrapperSingle(const __m128& op, TParams... params) {
+ TQType<TOpOut>::As(Value) =
+ func(TQType<TOpIn>::As(op), params...);
+ }
+};
+
+inline __m128i _mm_cvtps_epi32(__m128 a) {
+ /// vcvtq_s32_f32 rounds to zero, but we need to round to the nearest.
+ static const float32x4_t half = vdupq_n_f32(0.5f);
+ static const float32x4_t negHalf = vdupq_n_f32(-0.5f);
+ static const float32x4_t zero = vdupq_n_f32(0.0f);
+ const float32x4_t corrections = vbslq_f32(vcgeq_f32(a.AsFloat32x4, zero), half, negHalf);
+ __m128i res;
+ res.AsSi32x4 = vcvtq_s32_f32(vaddq_f32(a.AsFloat32x4, corrections));
+ return res;
+}
+
+using _mm_cvttps_epi32 =
+ TCvtF2SWrapperSingle<int32x4_t, float32x4_t,
+ decltype(vcvtq_s32_f32), vcvtq_s32_f32>;
+
+Y_FORCE_INLINE int
+_mm_movemask_ps(const __m128& op) {
+ uint32x4_t mask = {0x80000000, 0x80000000, 0x80000000, 0x80000000};
+ uint32x4_t bits = vandq_u32(op.AsUi32x4, mask);
+ int32x4_t shifts = {-31, -30, -29, -28};
+ bits = vshlq_u32(bits, shifts);
+ return vaddvq_u32(bits);
+}
+
+Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) {
+ return vgetq_lane_s64(a.AsSi64x2, 0);
+}
+
+static inline void _mm_pause() {
+ __asm__ ("YIELD");
+}
+
+static inline __m128 _mm_rsqrt_ps(__m128 a) {
+ __m128 res;
+ res.AsFloat32x4 = vrsqrteq_f32(a.AsFloat32x4);
+ return res;
+}
+
+inline float _mm_cvtss_f32(__m128 a) {
+ return a.AsFloat32x4[0];
+}
+
+inline __m128 _mm_cmpunord_ps(__m128 a, __m128 b) {
+ __m128 res;
+ res.AsUi32x4 = vorrq_u32(
+ vmvnq_u32(vceqq_f32(a.AsFloat32x4, a.AsFloat32x4)), //!< 0xffffffff for all nans in a.
+ vmvnq_u32(vceqq_f32(b.AsFloat32x4, b.AsFloat32x4)) //!< 0xffffffff all nans in b.
+ );
+ return res;
+}
+
+inline __m128 _mm_andnot_ps(__m128 a, __m128 b) {
+ __m128 res;
+ res.AsFloat32x4 = vandq_u32(vmvnq_u32(a.AsUi32x4), b.AsUi32x4);
+ return res;
+}
+
+inline void _mm_store_ss(float* p, __m128 a) {
+ *p = vgetq_lane_f32(a.AsFloat32x4, 0);
+}
+
+inline float vgetg_lane_f32_switch(float32x4_t a, ui8 b) {
+ switch (b & 0x3) {
+ case 0:
+ return vgetq_lane_f32(a, 0);
+ case 1:
+ return vgetq_lane_f32(a, 1);
+ case 2:
+ return vgetq_lane_f32(a, 2);
+ case 3:
+ return vgetq_lane_f32(a, 3);
+ }
+ return 0;
+}
+
+inline __m128 _mm_shuffle_ps(__m128 a, __m128 b, const ShuffleStruct4& shuf) {
+ __m128 ret;
+ ret.AsFloat32x4 = vmovq_n_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[0]));
+ ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(a.AsFloat32x4, shuf.x[1]), ret.AsFloat32x4, 1);
+ ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[2]), ret.AsFloat32x4, 2);
+ ret.AsFloat32x4 = vsetq_lane_f32(vgetg_lane_f32_switch(b.AsFloat32x4, shuf.x[3]), ret.AsFloat32x4, 3);
+ return ret;
+}
+
+inline __m128 _mm_or_ps(__m128 a, __m128 b) {
+ __m128 res;
+ res.AsUi32x4 = vorrq_u32(a.AsUi32x4, b.AsUi32x4);
+ return res;
+}
+
+inline __m128i _mm_sad_epu8(__m128i a, __m128i b) {
+ uint16x8_t t = vpaddlq_u8(vabdq_u8(a.AsUi8x16, b.AsUi8x16));
+ uint16_t r0 = t[0] + t[1] + t[2] + t[3];
+ uint16_t r4 = t[4] + t[5] + t[6] + t[7];
+ uint16x8_t r = vsetq_lane_u16(r0, vdupq_n_u16(0), 0);
+ __m128i ans;
+ ans.AsUi16x8 = vsetq_lane_u16(r4, r, 4);
+ return ans;
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epi8(__m128i a, __m128i b) {
+ __m128i ans;
+ ans.AsSi8x16 = vqsubq_s8(a.AsSi8x16, b.AsSi8x16);
+ return ans;
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epi16(__m128i a, __m128i b) {
+ __m128i ans;
+ ans.AsSi16x8 = vqsubq_s16(a.AsSi16x8, b.AsSi16x8);
+ return ans;
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epu8(__m128i a, __m128i b) {
+ __m128i ans;
+ ans.AsUi8x16 = vqsubq_u8(a.AsUi8x16, b.AsUi8x16);
+ return ans;
+}
+
+Y_FORCE_INLINE __m128i _mm_subs_epu16(__m128i a, __m128i b) {
+ __m128i ans;
+ ans.AsUi16x8 = vqsubq_u16(a.AsUi16x8, b.AsUi16x8);
+ return ans;
+}
+
+Y_FORCE_INLINE __m128d _mm_castsi128_pd(__m128i __A) {
+ return reinterpret_cast<__m128d&>(__A);
+}
+
+Y_FORCE_INLINE __m128i _mm_set_epi8(ui8 i15, ui8 i14, ui8 i13, ui8 i12, ui8 i11, ui8 i10, ui8 i9, ui8 i8,
+ ui8 i7, ui8 i6, ui8 i5, ui8 i4, ui8 i3, ui8 i2, ui8 i1, ui8 i0)
+{
+ int a0 = i0 | (i1<<8) | (i2<<16) | (i3<<24);
+ int a1 = i4 | (i5<<8) | (i6<<16) | (i7<<24);
+ int a2 = i8 | (i9<<8) | (i10<<16) | (i11<<24);
+ int a3 = i12 | (i13<<8) | (i14<<16) | (i15<<24);
+ return _mm_set_epi32(a3, a2, a1, a0);
+}
+
+Y_FORCE_INLINE __m128i _mm_max_epu8(__m128i a, __m128i b) {
+ __m128i ans;
+ ans.AsUi8x16 = vmaxq_u8(a.AsUi8x16, b.AsUi8x16);
+ return ans;
+}
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wuninitialized"
+Y_FORCE_INLINE __m128d _mm_undefined_pd(void) {
+ __m128d ans = ans;
+ return ans;
+}
+#pragma GCC diagnostic pop
+
+Y_FORCE_INLINE __m128d _mm_loadh_pd(__m128d a, const double* b) {
+ a[1] = *b;
+ return a;
+}
+
+Y_FORCE_INLINE __m128d _mm_loadl_pd(__m128d a, const double* b) {
+ a[0] = *b;
+ return a;
+}
+
+Y_FORCE_INLINE double _mm_cvtsd_f64(__m128d a) {
+ return a[0];
+}
+
+Y_FORCE_INLINE __m128d _mm_shuffle_pd(__m128d a, __m128d b, int mask) {
+ __m128d result;
+ const int litmsk = mask & 0x3;
+
+ if (litmsk == 0)
+ result = vzip1q_f64(a, b);
+ else if (litmsk == 1)
+ result = __builtin_shufflevector(a, b, 1, 2);
+ else if (litmsk == 2)
+ result = __builtin_shufflevector(a, b, 0, 3);
+ else
+ result = vzip2q_f64(a, b);
+ return result;
+}
diff --git a/library/cpp/sse/ut/test.cpp b/library/cpp/sse/ut/test.cpp
new file mode 100644
index 0000000000..33c999d284
--- /dev/null
+++ b/library/cpp/sse/ut/test.cpp
@@ -0,0 +1,2088 @@
+/*
+ Unittests for all SSE instrinsics translated to NEON instrinsics or
+ software implementation.
+ Should be tested both on Intel and ARM64.
+ */
+/* Author: Vitaliy Manushkin <agri@yandex-team.ru */
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/typetraits.h>
+#include <util/string/hex.h>
+#include <util/random/fast.h>
+#include <util/stream/output.h>
+
+#include <algorithm>
+#include <array>
+#include <limits>
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+template <typename TResult, typename TFunc, TFunc* func>
+struct T_mm_CallWrapper {
+ TResult Value;
+
+ template <typename... TParams>
+ T_mm_CallWrapper(TParams&&... params) {
+ Value = func(std::forward<TParams>(params)...);
+ }
+
+ operator TResult&() {
+ return Value;
+ }
+
+ operator const TResult&() const {
+ return Value;
+ }
+};
+
+#if defined(_arm64_)
+#include "library/cpp/sse/sse2neon.h"
+#elif defined(_i386_) || defined(_x86_64_)
+#include <xmmintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+#elif defined(_ppc64_)
+#include "library/cpp/sse/powerpc.h"
+#else
+#error "Unsupported platform"
+#endif
+
+#if defined(_arm64_)
+#define Wrap(T_mm_func) T_mm_func
+#define WrapF(T_mm_func) T_mm_func
+#define WrapD(T_mm_func) T_mm_func
+#elif defined(_ppc64_) || defined(_i386_) || defined(_x86_64_)
+#define Wrap(_mm_func) \
+ T_mm_CallWrapper<__m128i, decltype(_mm_func), _mm_func>
+#define WrapF(_mm_func) \
+ T_mm_CallWrapper<__m128, decltype(_mm_func), _mm_func>
+#define WrapD(_mm_func) \
+ T_mm_CallWrapper<__m128d, decltype(_mm_func), _mm_func>
+using int8x16_t = std::array<i8, 16>;
+using int16x8_t = std::array<i16, 8>;
+using int32x4_t = std::array<i32, 4>;
+using int64x2_t = std::array<i64, 2>;
+using uint8x16_t = std::array<ui8, 16>;
+using uint16x8_t = std::array<ui16, 8>;
+using uint32x4_t = std::array<ui32, 4>;
+using uint64x2_t = std::array<ui64, 2>;
+using float32x4_t = std::array<float, 4>;
+using float64x2_t = std::array<double, 2>;
+
+template <typename TVectorType>
+struct TQType {
+ static TVectorType As(__m128i param) {
+ TVectorType value;
+ _mm_storeu_si128((__m128i*)&value, param);
+ return value;
+ }
+ static TVectorType As(__m128 param) {
+ TVectorType value;
+ _mm_storeu_ps((float*)&value, param);
+ return value;
+ }
+ static TVectorType As(__m128d param) {
+ TVectorType value;
+ _mm_storeu_pd((double*)&value, param);
+ return value;
+ }
+};
+#endif
+
+template <typename TVectorType>
+struct TFuncLoad;
+template <typename TVectorType>
+struct TFuncStore;
+
+template <>
+struct TFuncLoad<__m128i> {
+ __m128i Value;
+
+ template <typename TPointer>
+ TFuncLoad(TPointer* ptr) {
+ Value = _mm_loadu_si128((__m128i*)ptr);
+ }
+
+ operator __m128i&() {
+ return Value;
+ }
+
+ operator const __m128i&() const {
+ return Value;
+ }
+};
+
+template <>
+struct TFuncLoad<__m128> {
+ __m128 Value;
+
+ template <typename TPointer>
+ TFuncLoad(TPointer* ptr) {
+ Value = _mm_loadu_ps((float*)ptr);
+ }
+
+ operator __m128&() {
+ return Value;
+ }
+
+ operator const __m128&() const {
+ return Value;
+ }
+};
+
+template <>
+struct TFuncLoad<__m128d> {
+ __m128d Value;
+
+ template <typename TPointer>
+ TFuncLoad(TPointer* ptr) {
+ Value = _mm_loadu_pd((double*)ptr);
+ }
+
+ operator __m128d&() {
+ return Value;
+ }
+
+ operator const __m128d&() const {
+ return Value;
+ }
+};
+
+template <>
+struct TFuncStore<__m128i> {
+ template <typename TPointer>
+ TFuncStore(TPointer* ptr, __m128i Value) {
+ _mm_storeu_si128((__m128i*)ptr, Value);
+ }
+};
+
+template <>
+struct TFuncStore<__m128> {
+ template <typename TPointer>
+ TFuncStore(TPointer* ptr, __m128 Value) {
+ _mm_storeu_ps((float*)ptr, Value);
+ }
+};
+
+class TSSEEmulTest: public TTestBase {
+private:
+ UNIT_TEST_SUITE(TSSEEmulTest);
+ UNIT_TEST(Test_mm_load_si128);
+ UNIT_TEST(Test_mm_loadu_si128);
+ UNIT_TEST(Test_mm_storeu_si128);
+ UNIT_TEST(Test_mm_loadu_si128_2);
+ UNIT_TEST(Test_mm_loadu_ps);
+ UNIT_TEST(Test_mm_storeu_ps);
+
+ UNIT_TEST(Test_mm_slli_epi16);
+ UNIT_TEST(Test_mm_slli_epi32);
+ UNIT_TEST(Test_mm_slli_epi64);
+ UNIT_TEST(Test_mm_slli_si128);
+
+ UNIT_TEST(Test_mm_srli_epi16);
+ UNIT_TEST(Test_mm_srli_epi32);
+ UNIT_TEST(Test_mm_srli_epi64);
+ UNIT_TEST(Test_mm_srli_si128);
+
+ UNIT_TEST(Test_mm_srai_epi16);
+ UNIT_TEST(Test_mm_srai_epi32);
+
+ UNIT_TEST(Test_mm_sll_epi16);
+ UNIT_TEST(Test_mm_sll_epi32);
+ UNIT_TEST(Test_mm_sll_epi64);
+
+ UNIT_TEST(Test_mm_srl_epi16);
+ UNIT_TEST(Test_mm_srl_epi32);
+ UNIT_TEST(Test_mm_srl_epi64);
+
+ UNIT_TEST(Test_mm_add_epi16);
+ UNIT_TEST(Test_mm_add_epi32);
+ UNIT_TEST(Test_mm_add_epi64);
+ UNIT_TEST(Test_mm_add_ps);
+ UNIT_TEST(Test_mm_add_pd);
+
+ UNIT_TEST(Test_mm_madd_epi16);
+
+ UNIT_TEST(Test_mm_sub_epi16);
+ UNIT_TEST(Test_mm_sub_epi32);
+ UNIT_TEST(Test_mm_sub_epi64);
+ UNIT_TEST(Test_mm_sub_ps);
+ UNIT_TEST(Test_mm_sub_pd);
+
+ UNIT_TEST(Test_mm_mul_ps);
+ UNIT_TEST(Test_mm_mul_pd);
+ UNIT_TEST(Test_mm_div_ps);
+ UNIT_TEST(Test_mm_div_pd);
+ UNIT_TEST(Test_mm_max_ps);
+ UNIT_TEST(Test_mm_min_ps);
+ UNIT_TEST(Test_mm_and_ps);
+
+ UNIT_TEST(Test_mm_unpacklo_epi8);
+ UNIT_TEST(Test_mm_unpackhi_epi8);
+ UNIT_TEST(Test_mm_unpacklo_epi16);
+ UNIT_TEST(Test_mm_unpackhi_epi16);
+ UNIT_TEST(Test_mm_unpacklo_epi32);
+ UNIT_TEST(Test_mm_unpackhi_epi32);
+ UNIT_TEST(Test_mm_unpacklo_epi64);
+ UNIT_TEST(Test_mm_unpackhi_epi64);
+
+ UNIT_TEST(Test_mm_or_si128);
+ UNIT_TEST(Test_mm_and_si128);
+ UNIT_TEST(Test_mm_andnot_si128);
+
+ UNIT_TEST(Test_mm_cmpeq_epi8);
+ UNIT_TEST(Test_mm_cmpeq_epi16);
+ UNIT_TEST(Test_mm_cmpeq_epi32);
+ UNIT_TEST(Test_mm_cmpeq_ps);
+
+ UNIT_TEST(Test_mm_cmpgt_epi8);
+ UNIT_TEST(Test_mm_cmpgt_epi16);
+ UNIT_TEST(Test_mm_cmpgt_epi32);
+ UNIT_TEST(Test_mm_cmpgt_ps);
+
+ UNIT_TEST(Test_mm_cmplt_epi8);
+ UNIT_TEST(Test_mm_cmplt_epi16);
+ UNIT_TEST(Test_mm_cmplt_epi32);
+
+ UNIT_TEST(Test_mm_set1_epi8);
+ UNIT_TEST(Test_mm_set1_epi16);
+ UNIT_TEST(Test_mm_set1_epi32);
+ UNIT_TEST(Test_mm_set1_ps);
+ UNIT_TEST(Test_mm_set_ps1);
+
+ UNIT_TEST(Test_mm_setzero_si128);
+ UNIT_TEST(Test_mm_setzero_ps);
+ UNIT_TEST(Test_mm_setzero_pd);
+
+ UNIT_TEST(Test_mm_storel_epi64);
+ UNIT_TEST(Test_mm_loadl_epi64);
+
+ UNIT_TEST(Test_mm_loadl_pd);
+ UNIT_TEST(Test_mm_loadh_pd);
+ UNIT_TEST(Test_mm_cvtsd_f64);
+
+ UNIT_TEST(Test_mm_shuffle_epi32);
+ UNIT_TEST(Test_mm_movemask_epi8);
+ UNIT_TEST(Test_mm_cvtsi128_si32);
+ UNIT_TEST(Test_mm_cvtsi128_si64);
+
+ UNIT_TEST(Test_mm_set_epi16);
+ UNIT_TEST(Test_mm_set_epi32);
+ UNIT_TEST(Test_mm_set_ps);
+ UNIT_TEST(Test_mm_set_pd);
+
+ UNIT_TEST(Test_mm_cvtsi32_si128);
+ UNIT_TEST(Test_mm_cvtsi64_si128);
+
+ UNIT_TEST(Test_mm_packs_epi16);
+ UNIT_TEST(Test_mm_packs_epi32);
+ UNIT_TEST(Test_mm_packus_epi16);
+
+ UNIT_TEST(Test_mm_extract_epi16);
+ UNIT_TEST(Test_mm_extract_epi8);
+ UNIT_TEST(Test_mm_extract_epi32);
+ UNIT_TEST(Test_mm_extract_epi64);
+
+ UNIT_TEST(Test_MM_TRANSPOSE4_PS);
+ UNIT_TEST(Test_mm_movemask_ps);
+ UNIT_TEST(Test_mm_movemask_ps_2);
+
+ UNIT_TEST(Test_mm_cvtepi32_ps);
+ UNIT_TEST(Test_mm_cvtps_epi32);
+ UNIT_TEST(Test_mm_cvttps_epi32);
+
+ UNIT_TEST(Test_mm_castsi128_ps);
+ UNIT_TEST(Test_mm_castps_si128);
+
+ UNIT_TEST(Test_mm_mul_epu32);
+
+ UNIT_TEST(Test_mm_cmpunord_ps);
+ UNIT_TEST(Test_mm_andnot_ps);
+ UNIT_TEST(Test_mm_shuffle_ps);
+ UNIT_TEST(Test_mm_shuffle_pd);
+ UNIT_TEST(Test_mm_or_ps);
+ UNIT_TEST(Test_mm_store_ss);
+ UNIT_TEST(Test_mm_store_ps);
+ UNIT_TEST(Test_mm_storeu_pd);
+ UNIT_TEST(Test_mm_loadu_pd);
+ UNIT_TEST(Test_mm_rsqrt_ps);
+ UNIT_TEST(Test_matrixnet_powerpc);
+
+ UNIT_TEST_SUITE_END();
+
+public:
+ void Test_mm_load_si128();
+ void Test_mm_loadu_si128();
+ void Test_mm_storeu_si128();
+ void Test_mm_loadu_si128_2();
+ void Test_mm_loadu_ps();
+ void Test_mm_storeu_ps();
+
+ template <typename TElem, int bits, int elemCount,
+ typename TFunc, typename TShifter, typename TOp, typename TElemFunc>
+ void Test_mm_shifter_epiXX();
+
+ enum class EDirection {
+ Left,
+ Right
+ };
+
+ struct TShiftRes {
+ __m128i Value[17];
+ };
+
+ void Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo);
+
+ void Test_mm_slli_epi16();
+ void Test_mm_slli_epi32();
+ void Test_mm_slli_epi64();
+ void Test_mm_slli_si128();
+
+ void Test_mm_srli_epi16();
+ void Test_mm_srli_epi32();
+ void Test_mm_srli_epi64();
+ void Test_mm_srli_si128();
+
+ void Test_mm_srai_epi16();
+ void Test_mm_srai_epi32();
+
+ void Test_mm_sll_epi16();
+ void Test_mm_sll_epi32();
+ void Test_mm_sll_epi64();
+
+ void Test_mm_srl_epi16();
+ void Test_mm_srl_epi32();
+ void Test_mm_srl_epi64();
+
+ void Test_mm_add_epi8();
+ void Test_mm_add_epi16();
+ void Test_mm_add_epi32();
+ void Test_mm_add_epi64();
+ void Test_mm_add_ps();
+ void Test_mm_add_pd();
+
+ void Test_mm_madd_epi16();
+
+ void Test_mm_sub_epi8();
+ void Test_mm_sub_epi16();
+ void Test_mm_sub_epi32();
+ void Test_mm_sub_epi64();
+ void Test_mm_sub_ps();
+ void Test_mm_sub_pd();
+
+ void Test_mm_mul_ps();
+ void Test_mm_mul_pd();
+ void Test_mm_div_ps();
+ void Test_mm_div_pd();
+ void Test_mm_max_ps();
+ void Test_mm_min_ps();
+ void Test_mm_and_ps();
+
+ template <typename TElem, int bits, int elemCount, int shift,
+ typename TFunc, typename TOp>
+ void Test_mm_unpack_epiXX();
+ void Test_mm_unpacklo_epi8();
+ void Test_mm_unpackhi_epi8();
+ void Test_mm_unpacklo_epi16();
+ void Test_mm_unpackhi_epi16();
+ void Test_mm_unpacklo_epi32();
+ void Test_mm_unpackhi_epi32();
+ void Test_mm_unpacklo_epi64();
+ void Test_mm_unpackhi_epi64();
+
+ template <typename TElem, unsigned elemCount,
+ typename TFunc, typename TElemFunc,
+ typename TOp, typename TVectorType = __m128i>
+ void Test_mm_dualop();
+
+ template <typename TElem, unsigned elemCount,
+ typename TFunc, typename TElemFunc,
+ typename TOp, typename TVectorType = __m128i>
+ void Test_mm_dualcmp();
+
+ void Test_mm_or_si128();
+ void Test_mm_and_si128();
+ void Test_mm_andnot_si128();
+
+ void Test_mm_cmpeq_epi8();
+ void Test_mm_cmpeq_epi16();
+ void Test_mm_cmpeq_epi32();
+ void Test_mm_cmpeq_ps();
+
+ void Test_mm_cmpgt_epi8();
+ void Test_mm_cmpgt_epi16();
+ void Test_mm_cmpgt_epi32();
+ void Test_mm_cmpgt_ps();
+
+ void Test_mm_cmplt_epi8();
+ void Test_mm_cmplt_epi16();
+ void Test_mm_cmplt_epi32();
+
+ template <typename TElem, int elemCount,
+ typename TFunc, typename TOp, typename TVectorType>
+ void Test_mm_setter_epiXX();
+ void Test_mm_set1_epi8();
+ void Test_mm_set1_epi16();
+ void Test_mm_set1_epi32();
+ void Test_mm_set1_ps();
+ void Test_mm_set_ps1();
+
+ void Test_mm_setzero_si128();
+ void Test_mm_setzero_ps();
+ void Test_mm_setzero_pd();
+
+ void Test_mm_loadl_epi64();
+ void Test_mm_storel_epi64();
+
+ void Test_mm_loadl_pd();
+ void Test_mm_loadh_pd();
+ void Test_mm_cvtsd_f64();
+
+ void Test_mm_shuffle_epi32();
+ void Test_mm_movemask_epi8();
+ void Test_mm_cvtsi128_si32();
+ void Test_mm_cvtsi128_si64();
+
+ void Test_mm_set_epi16();
+ void Test_mm_set_epi32();
+ void Test_mm_set_ps();
+ void Test_mm_set_pd();
+
+ void Test_mm_cvtsi32_si128();
+ void Test_mm_cvtsi64_si128();
+
+ template <typename TElem, typename TNarrow, unsigned elemCount,
+ typename TFunc>
+ void Test_mm_packs_epiXX();
+ void Test_mm_packs_epi16();
+ void Test_mm_packs_epi32();
+ void Test_mm_packus_epi16();
+
+ void Test_mm_extract_epi16();
+ void Test_mm_extract_epi8();
+ void Test_mm_extract_epi32();
+ void Test_mm_extract_epi64();
+
+ void Test_MM_TRANSPOSE4_PS();
+ void Test_mm_movemask_ps();
+ void Test_mm_movemask_ps_2();
+
+ template <typename TFrom, typename TTo, unsigned elemCount,
+ typename TLoadVector, typename TResultVector,
+ typename TElemFunc, typename TFunc, typename TOp>
+ void Test_mm_convertop();
+ void Test_mm_cvtepi32_ps();
+ void Test_mm_cvtps_epi32();
+ void Test_mm_cvttps_epi32();
+
+ template <typename TLoadVector, typename TCastVector,
+ typename TFunc, TFunc* func>
+ void Test_mm_castXX();
+ void Test_mm_castsi128_ps();
+ void Test_mm_castps_si128();
+
+ void Test_mm_mul_epu32();
+
+ void Test_mm_cmpunord_ps();
+ void Test_mm_store_ss();
+ void Test_mm_store_ps();
+ void Test_mm_storeu_pd();
+ void Test_mm_andnot_ps();
+ void Test_mm_shuffle_ps();
+ void Test_mm_shuffle_pd();
+ void Test_mm_or_ps();
+ void Test_mm_loadu_pd();
+ void Test_mm_rsqrt_ps();
+ void Test_mm_rsqrt_ss();
+ void Test_matrixnet_powerpc();
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TSSEEmulTest);
+
+void TSSEEmulTest::Test_mm_load_si128() {
+ alignas(16) char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ __m128i value = _mm_load_si128((__m128i*)&data);
+ UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[0], 0xAABB2211CCFF00AAUL);
+ UNIT_ASSERT_EQUAL(TQType<uint64x2_t>::As(value)[1], 0x1C66775588449933UL);
+}
+
+void TSSEEmulTest::Test_mm_loadu_si128() {
+ alignas(16) char data[17] = {
+ '\x66',
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ UNIT_ASSERT((ui64(&data[1]) & 0x1) == 0x1);
+ __m128i value = _mm_loadu_si128((__m128i*)&data[1]);
+ UNIT_ASSERT(TQType<uint64x2_t>::As(value)[0] == 0xAABB2211CCFF00AAUL);
+ UNIT_ASSERT(TQType<uint64x2_t>::As(value)[1] == 0x1C66775588449933UL);
+}
+
+void TSSEEmulTest::Test_mm_storeu_si128() {
+ alignas(16) unsigned char stub[32] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
+ };
+
+ alignas(16) unsigned char value[16] = {
+ 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+ 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf
+ };
+
+ const __m128i val = _mm_loadu_si128((__m128i*)&value[0]);
+
+ for (size_t shift = 0; shift != 17; ++shift) {
+ alignas(16) unsigned char res[sizeof(stub)];
+ memcpy(res, stub, sizeof(res));
+
+ _mm_storeu_si128((__m128i*)&res[shift], val);
+
+
+ alignas(16) unsigned char etalon[sizeof(stub)];
+ memcpy(etalon, stub, sizeof(etalon));
+ for (size_t i = 0; i != sizeof(value); ++i) {
+ etalon[shift + i] = value[i];
+ }
+
+ for (size_t i = 0; i != sizeof(etalon) / sizeof(etalon[0]); ++i) {
+ UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, 32) << " vs etalon: " << HexEncode(etalon, 32));
+ }
+ }
+
+}
+
+
+void TSSEEmulTest::Test_mm_loadu_si128_2() {
+ alignas(16) unsigned char stub[32] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f
+ };
+
+ for (size_t shift = 0; shift != 17; ++shift) {
+ const __m128i val = _mm_loadu_si128((const __m128i*)&stub[shift]);
+ alignas(16) unsigned char res[16];
+ _mm_store_si128((__m128i*)res, val);
+
+ for (size_t i = 0; i != 16; ++i) {
+ UNIT_ASSERT_EQUAL_C(res[i], stub[i + shift], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16));
+ }
+ }
+}
+
+
+void TSSEEmulTest::Test_mm_loadu_ps() {
+ alignas(16) float stub[8] = {
+ 0.f, 1.f, 2.f, 3.f,
+ 4.f, 5.f, 6.f, 7.f
+ };
+
+ for (size_t shift = 0; shift != 5; ++shift) {
+ const __m128 val = _mm_loadu_ps(&stub[shift]);
+ alignas(16) float res[4];
+ _mm_store_ps(res, val);
+
+ for (size_t i = 0; i != 4; ++i) {
+ UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16));
+ }
+ }
+}
+
+
+void TSSEEmulTest::Test_mm_storeu_ps() {
+ alignas(16) float stub[8] = {
+ 0.f, 1.f, 2.f, 3.f,
+ 4.f, 5.f, 6.f, 7.f
+ };
+
+ alignas(16) float value[4] = {
+ 100.f, 101.f, 102.f, 103.f
+ };
+ const __m128 val = _mm_load_ps(value);
+
+ for (size_t shift = 0; shift != 5; ++shift) {
+ alignas(16) float res[sizeof(stub) / sizeof(stub[0])];
+ memcpy(res, stub, sizeof(stub));
+
+ _mm_storeu_ps(&res[shift], val);
+
+ float etalon[sizeof(stub) / sizeof(stub[0])];
+ memcpy(etalon, stub, sizeof(stub));
+ for (size_t i = 0; i != 4; ++i) {
+ etalon[i + shift] = value[i];
+ }
+
+ for (size_t i = 0; i != sizeof(stub) / sizeof(stub[0]); ++i) {
+ UNIT_ASSERT_EQUAL_C(res[i], etalon[i], "res: " << HexEncode(res, sizeof(res)) << " vs etalon: " << HexEncode(etalon, sizeof(etalon)));
+ }
+ }
+}
+
+template<typename C>
+C MakeNumber(unsigned number);
+
+template<>
+__m128i MakeNumber<__m128i>(unsigned number) {
+ char data[16] = {0};
+ memcpy(data, &number, sizeof(number));
+
+ return _mm_loadu_si128((__m128i*)data);
+}
+
+template<>
+unsigned MakeNumber<unsigned>(unsigned number) {
+ return number;
+}
+
+template <typename TElem, int bits, int elemCount,
+ typename TFunc, typename TShifter, typename TOp, typename TElemFunc>
+void TSSEEmulTest::Test_mm_shifter_epiXX() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ TElem* dataw = reinterpret_cast<TElem*>(&data);
+
+ __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ for (unsigned shifter = 0; shifter <= bits; ++shifter) {
+ TElem shiftedData[elemCount];
+ for (unsigned i = 0; i < elemCount; ++i) {
+ shiftedData[i] = TElemFunc::Call(dataw[i], shifter);
+ }
+
+ const TShifter adhoc_shifter = MakeNumber<TShifter>(shifter);
+
+ __m128i result = TFunc(value, adhoc_shifter);
+
+ for (unsigned i = 0; i < elemCount; ++i) {
+ UNIT_ASSERT_EQUAL(shiftedData[i], TQType<TOp>::As(result)[i]);
+ }
+ }
+}
+
+
+void TSSEEmulTest::Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo) {
+ const char data[48] = {
+ '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C',
+ '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'
+ };
+
+ const __m128i a = _mm_loadu_si128((__m128i*)(data + 16));
+ const TShiftRes res = foo(a);
+ for (int shift = 0; shift <= 16; ++shift) {
+ const int etalon_offset = 16 + (direction == EDirection::Left ? -shift : shift); //!< specific to little endian byte order.
+ const char* etalon = data + etalon_offset;
+ const char* res_bytes = (const char*)&res.Value[shift];
+
+ for (size_t byte = 0; byte != 16; ++byte) {
+ UNIT_ASSERT_EQUAL(etalon[byte], res_bytes[byte]);
+ }
+ }
+}
+
+template <typename TElem>
+struct THelperASHR {
+ static TElem Call(const TElem op, const int shift) {
+ constexpr int nBitsInOp = sizeof(op) * CHAR_BIT;
+ if (op < 0) {
+ // Arithmetic shift propagates sign bit to the right
+ // while operator>> is implementation defined for negative values,
+ // so we can't use it as a reference implementation
+ // and we need to write some standard consistent code.
+ typedef TFixedWidthUnsignedInt<TElem> TUnsignedElem;
+ TUnsignedElem uOp(op);
+ const TUnsignedElem signBit = TUnsignedElem(1) << (nBitsInOp - 1);
+ Y_ENSURE(shift >= 0);
+ for (int i = 0; i != shift; ++i) {
+ uOp = signBit | (uOp >> 1);
+ }
+ // unsigned -> signed conversion is also implementation defined, so we need to use some other method.
+ return reinterpret_cast<TElem&>(uOp);
+ }
+ return shift < nBitsInOp ? op >> shift : 0;
+ }
+};
+
+template <typename TElem>
+struct THelperSHR {
+ static TElem Call(const TElem op, const int shift) {
+ constexpr int nBitsInOp = sizeof(op) * CHAR_BIT;
+ return shift < nBitsInOp ? op >> shift : 0;
+ }
+};
+
+void TSSEEmulTest::Test_mm_srli_epi16() {
+ Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_srli_epi16), unsigned, uint16x8_t,
+ THelperSHR<ui16>>();
+}
+
+void TSSEEmulTest::Test_mm_srli_epi32() {
+ Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_srli_epi32), unsigned, uint32x4_t,
+ THelperSHR<ui32>>();
+}
+
+void TSSEEmulTest::Test_mm_srli_epi64() {
+ Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_srli_epi64), unsigned, uint64x2_t,
+ THelperSHR<ui64>>();
+}
+
+template <typename TElem>
+struct THelperSHL {
+ static TElem Call(const TElem op, const int shift) {
+ constexpr int nBitsInOp = sizeof(op) * CHAR_BIT;
+ return shift < nBitsInOp ? op << shift : 0;
+ }
+};
+
+void TSSEEmulTest::Test_mm_slli_epi16() {
+ Test_mm_shifter_epiXX<ui16, 16, 8, Wrap(_mm_slli_epi16), unsigned, uint16x8_t,
+ THelperSHL<ui16>>();
+}
+
+void TSSEEmulTest::Test_mm_slli_epi32() {
+ Test_mm_shifter_epiXX<ui32, 32, 4, Wrap(_mm_slli_epi32), unsigned, uint32x4_t,
+ THelperSHL<ui32>>();
+}
+
+void TSSEEmulTest::Test_mm_slli_epi64() {
+ Test_mm_shifter_epiXX<ui64, 64, 2, Wrap(_mm_slli_epi64), unsigned, uint64x2_t,
+ THelperSHL<ui64>>();
+}
+
+void TSSEEmulTest::Test_mm_slli_si128() {
+ Test_mm_byte_shifter(EDirection::Left, [] (__m128i a) -> TShiftRes {
+ TShiftRes res;
+ res.Value[0] = _mm_slli_si128(a, 0);
+ res.Value[1] = _mm_slli_si128(a, 1);
+ res.Value[2] = _mm_slli_si128(a, 2);
+ res.Value[3] = _mm_slli_si128(a, 3);
+ res.Value[4] = _mm_slli_si128(a, 4);
+ res.Value[5] = _mm_slli_si128(a, 5);
+ res.Value[6] = _mm_slli_si128(a, 6);
+ res.Value[7] = _mm_slli_si128(a, 7);
+ res.Value[8] = _mm_slli_si128(a, 8);
+ res.Value[9] = _mm_slli_si128(a, 9);
+ res.Value[10] = _mm_slli_si128(a, 10);
+ res.Value[11] = _mm_slli_si128(a, 11);
+ res.Value[12] = _mm_slli_si128(a, 12);
+ res.Value[13] = _mm_slli_si128(a, 13);
+ res.Value[14] = _mm_slli_si128(a, 14);
+ res.Value[15] = _mm_slli_si128(a, 15);
+ res.Value[16] = _mm_slli_si128(a, 16);
+
+ return res;
+ });
+}
+
+void TSSEEmulTest::Test_mm_srl_epi16() {
+ Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi16), _mm_srl_epi16>, __m128i, uint16x8_t,
+ THelperSHR<ui16>>();
+}
+
+void TSSEEmulTest::Test_mm_srl_epi32() {
+ Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi32), _mm_srl_epi32>, __m128i, uint32x4_t,
+ THelperSHR<ui32>>();
+}
+
+void TSSEEmulTest::Test_mm_srl_epi64() {
+ Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_srl_epi64), _mm_srl_epi64>, __m128i, uint64x2_t,
+ THelperSHR<ui64>>();
+}
+
+void TSSEEmulTest::Test_mm_srai_epi16() {
+ Test_mm_shifter_epiXX<i16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi16), _mm_srai_epi16>, unsigned, int16x8_t,
+ THelperASHR<i16>>();
+}
+
+void TSSEEmulTest::Test_mm_srai_epi32() {
+ Test_mm_shifter_epiXX<i32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_srai_epi32), _mm_srai_epi32>, unsigned, int32x4_t,
+ THelperASHR<i32>>();
+}
+
+void TSSEEmulTest::Test_mm_srli_si128() {
+ Test_mm_byte_shifter(EDirection::Right, [](__m128i a) -> TShiftRes {
+ TShiftRes res;
+ res.Value[0] = _mm_srli_si128(a, 0);
+ res.Value[1] = _mm_srli_si128(a, 1);
+ res.Value[2] = _mm_srli_si128(a, 2);
+ res.Value[3] = _mm_srli_si128(a, 3);
+ res.Value[4] = _mm_srli_si128(a, 4);
+ res.Value[5] = _mm_srli_si128(a, 5);
+ res.Value[6] = _mm_srli_si128(a, 6);
+ res.Value[7] = _mm_srli_si128(a, 7);
+ res.Value[8] = _mm_srli_si128(a, 8);
+ res.Value[9] = _mm_srli_si128(a, 9);
+ res.Value[10] = _mm_srli_si128(a, 10);
+ res.Value[11] = _mm_srli_si128(a, 11);
+ res.Value[12] = _mm_srli_si128(a, 12);
+ res.Value[13] = _mm_srli_si128(a, 13);
+ res.Value[14] = _mm_srli_si128(a, 14);
+ res.Value[15] = _mm_srli_si128(a, 15);
+ res.Value[16] = _mm_srli_si128(a, 16);
+
+ return res;
+ });
+}
+
+void TSSEEmulTest::Test_mm_sll_epi16() {
+ Test_mm_shifter_epiXX<ui16, 16, 8, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi16), _mm_sll_epi16>, __m128i, uint16x8_t,
+ THelperSHL<ui16>>();
+}
+
+void TSSEEmulTest::Test_mm_sll_epi32() {
+ Test_mm_shifter_epiXX<ui32, 32, 4, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi32), _mm_sll_epi32>, __m128i, uint32x4_t,
+ THelperSHL<ui32>>();
+}
+
+void TSSEEmulTest::Test_mm_sll_epi64() {
+ Test_mm_shifter_epiXX<ui64, 64, 2, T_mm_CallWrapper<__m128i, decltype(_mm_sll_epi64), _mm_sll_epi64>, __m128i, uint64x2_t,
+ THelperSHL<ui64>>();
+}
+
+template <typename TElem>
+struct THelperAdd {
+ static TElem Call(const TElem op1, const TElem op2) {
+ return op1 + op2;
+ }
+};
+
+void TSSEEmulTest::Test_mm_add_epi16() {
+ Test_mm_dualop<ui16, 8, Wrap(_mm_add_epi16), THelperAdd<ui16>, uint16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_add_epi32() {
+ Test_mm_dualop<ui32, 4, Wrap(_mm_add_epi32), THelperAdd<ui32>, uint32x4_t>();
+}
+
+void TSSEEmulTest::Test_mm_add_epi64() {
+ Test_mm_dualop<ui64, 2, Wrap(_mm_add_epi64), THelperAdd<ui64>, uint64x2_t>();
+}
+
+void TSSEEmulTest::Test_mm_add_ps() {
+ Test_mm_dualop<float, 2, WrapF(_mm_add_ps),
+ THelperAdd<float>, float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_add_pd() {
+ Test_mm_dualop<double, 2, WrapD(_mm_add_pd),
+ THelperAdd<double>, float64x2_t, __m128d>();
+}
+
+void TSSEEmulTest::Test_mm_madd_epi16() {
+ alignas(16) const char data1[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'
+ };
+ alignas(16) const char data2[16] = {
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'
+ };
+
+ const __m128i value1 = TFuncLoad<__m128i>(&data1);
+ const __m128i value2 = TFuncLoad<__m128i>(&data2);
+ const __m128i res = _mm_madd_epi16(value1, value2);
+
+ const i16* dataw1 = reinterpret_cast<const i16*>(&data1);
+ const i16* dataw2 = reinterpret_cast<const i16*>(&data2);
+
+ for (size_t i = 0; i != 4; ++i) {
+ const size_t dataIdx = i * 2;
+ const i32 etalonResult = (i32) dataw1[dataIdx] * (i32) dataw2[dataIdx] + (i32) dataw1[dataIdx + 1] * (i32) dataw2[dataIdx + 1];
+ const i32 value = TQType<int32x4_t>::As(res)[i];
+ UNIT_ASSERT_EQUAL(value, etalonResult);
+ }
+}
+
+
+template <typename TElem>
+struct THelperSub {
+ static TElem Call(const TElem op1, const TElem op2) {
+ return op1 - op2;
+ }
+};
+
+void TSSEEmulTest::Test_mm_sub_epi16() {
+ Test_mm_dualop<ui16, 8, Wrap(_mm_sub_epi16), THelperSub<ui16>, uint16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_sub_epi32() {
+ Test_mm_dualop<ui32, 4, Wrap(_mm_sub_epi32), THelperSub<ui32>, uint32x4_t>();
+}
+
+void TSSEEmulTest::Test_mm_sub_epi64() {
+ Test_mm_dualop<ui64, 2, Wrap(_mm_sub_epi64), THelperSub<ui64>, uint64x2_t>();
+}
+
+void TSSEEmulTest::Test_mm_sub_ps() {
+ Test_mm_dualop<float, 4, WrapF(_mm_sub_ps), THelperSub<float>,
+ float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_sub_pd() {
+ Test_mm_dualop<double, 2, WrapD(_mm_sub_pd), THelperSub<double>,
+ float64x2_t, __m128d>();
+}
+
+void TSSEEmulTest::Test_mm_mul_ps() {
+ struct THelper {
+ static float Call(const float op1, const float op2) {
+ return op1 * op2;
+ }
+ };
+ Test_mm_dualop<float, 4, WrapF(_mm_mul_ps), THelper, float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_mul_pd() {
+ struct THelper {
+ static double Call(const double op1, const double op2) {
+ return op1 * op2;
+ }
+ };
+ Test_mm_dualop<double, 2, WrapD(_mm_mul_pd), THelper, float64x2_t, __m128d>();
+}
+
+void TSSEEmulTest::Test_mm_div_ps() {
+ struct THelper {
+ static float Call(const float op1, const float op2) {
+ return op1 / op2;
+ }
+ };
+ Test_mm_dualop<float, 4, WrapF(_mm_div_ps), THelper, float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_div_pd() {
+ struct THelper {
+ static double Call(const double op1, const double op2) {
+ return op1 / op2;
+ }
+ };
+ Test_mm_dualop<double, 2, WrapD(_mm_div_pd), THelper, float64x2_t, __m128d>();
+}
+
+void TSSEEmulTest::Test_mm_max_ps() {
+ struct THelper {
+ static float Call(const float op1, const float op2) {
+ return std::max(op1, op2);
+ }
+ };
+ Test_mm_dualop<float, 4, WrapF(_mm_max_ps), THelper, float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_min_ps() {
+ struct THelper {
+ static float Call(const float op1, const float op2) {
+ return std::min(op1, op2);
+ }
+ };
+ Test_mm_dualop<float, 4, WrapF(_mm_min_ps), THelper, float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_and_ps() {
+ struct THelper {
+ static float Call(const float op1, const float op2) {
+ union Cast {
+ unsigned int AsUInt;
+ float AsFloat;
+ };
+ Cast v1, v2, result;
+ v1.AsFloat = op1;
+ v2.AsFloat = op2;
+ result.AsUInt = v1.AsUInt & v2.AsUInt;
+ return result.AsFloat;
+ }
+ };
+ Test_mm_dualcmp<float, 4, WrapF(_mm_and_ps),
+ THelper, float32x4_t, __m128>();
+}
+
+template <typename TElem, int bits, int elemCount, int shift,
+ typename TFunc, typename TOp>
+void TSSEEmulTest::Test_mm_unpack_epiXX() {
+ char data1[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ char data2[16] = {
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
+ TElem* dataw1 = reinterpret_cast<TElem*>(&data1);
+ TElem* dataw2 = reinterpret_cast<TElem*>(&data2);
+
+ __m128i value1 = _mm_loadu_si128((__m128i*)&data1);
+ __m128i value2 = _mm_loadu_si128((__m128i*)&data2);
+
+ TElem zippedData[elemCount];
+ for (unsigned i = 0; i < elemCount / 2; ++i) {
+ zippedData[i * 2] = dataw1[i + shift];
+ zippedData[i * 2 + 1] = dataw2[i + shift];
+ }
+ __m128i result = TFunc(value1, value2);
+
+ for (unsigned i = 0; i < elemCount / 2; ++i) {
+ UNIT_ASSERT_EQUAL(zippedData[i * 2], TQType<TOp>::As(result)[i * 2]);
+ UNIT_ASSERT_EQUAL(zippedData[i * 2 + 1],
+ TQType<TOp>::As(result)[i * 2 + 1]);
+ }
+}
+
+void TSSEEmulTest::Test_mm_unpacklo_epi8() {
+ Test_mm_unpack_epiXX<ui8, 8, 16, 0, Wrap(_mm_unpacklo_epi8), uint8x16_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpackhi_epi8() {
+ Test_mm_unpack_epiXX<ui8, 8, 16, 8, Wrap(_mm_unpackhi_epi8), uint8x16_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpacklo_epi16() {
+ Test_mm_unpack_epiXX<ui16, 16, 8, 0, Wrap(_mm_unpacklo_epi16), uint16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpackhi_epi16() {
+ Test_mm_unpack_epiXX<ui16, 16, 8, 4, Wrap(_mm_unpackhi_epi16), uint16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpacklo_epi32() {
+ Test_mm_unpack_epiXX<ui32, 32, 4, 0, Wrap(_mm_unpacklo_epi32), uint32x4_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpackhi_epi32() {
+ Test_mm_unpack_epiXX<ui32, 32, 4, 2, Wrap(_mm_unpackhi_epi32), uint32x4_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpacklo_epi64() {
+ Test_mm_unpack_epiXX<ui64, 64, 2, 0, Wrap(_mm_unpacklo_epi64), uint64x2_t>();
+}
+
+void TSSEEmulTest::Test_mm_unpackhi_epi64() {
+ Test_mm_unpack_epiXX<ui64, 64, 2, 1, Wrap(_mm_unpackhi_epi64), uint64x2_t>();
+}
+
+template <typename TElem, unsigned elemCount,
+ typename TFunc, typename TElemFunc,
+ typename TOp, typename TVectorType>
+void TSSEEmulTest::Test_mm_dualop() {
+ char data1[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ char data2[16] = {
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
+ TElem* dataw1 = reinterpret_cast<TElem*>(&data1);
+ TElem* dataw2 = reinterpret_cast<TElem*>(&data2);
+
+ TVectorType value1 = TFuncLoad<TVectorType>(&data1);
+ TVectorType value2 = TFuncLoad<TVectorType>(&data2);
+
+ TElem procData[elemCount];
+ for (unsigned i = 0; i < elemCount; ++i) {
+ procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]);
+ }
+ TVectorType result = TFunc(value1, value2);
+
+ for (unsigned i = 0; i < elemCount; ++i) {
+ UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]);
+ }
+}
+
+/* This is almost the same as Test_mm_dualop,
+ but different data1 and data2 */
+template <typename TElem, unsigned elemCount,
+ typename TFunc, typename TElemFunc,
+ typename TOp, typename TVectorType>
+void TSSEEmulTest::Test_mm_dualcmp() {
+ char data1[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x66', '\x77', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C'};
+ char data2[16] = {
+ '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
+ TElem* dataw1 = reinterpret_cast<TElem*>(&data1);
+ TElem* dataw2 = reinterpret_cast<TElem*>(&data2);
+
+ TVectorType value1 = TFuncLoad<TVectorType>(&data1);
+ TVectorType value2 = TFuncLoad<TVectorType>(&data2);
+
+ TElem procData[elemCount];
+ for (unsigned i = 0; i < elemCount; ++i) {
+ procData[i] = TElemFunc::Call(dataw1[i], dataw2[i]);
+ }
+ TVectorType result = TFunc(value1, value2);
+
+ for (unsigned i = 0; i < elemCount; ++i) {
+ /* memcmp is for compare to invalid floats in results */
+ const TElem value = TQType<TOp>::As(result)[i];
+ UNIT_ASSERT(memcmp(&(procData[i]), &value, sizeof(TElem)) == 0);
+ }
+}
+
+void TSSEEmulTest::Test_mm_or_si128() {
+ struct THelper {
+ static ui64 Call(const ui64 op1, const ui64 op2) {
+ return op1 | op2;
+ }
+ };
+
+ Test_mm_dualop<ui64, 2, Wrap(_mm_or_si128), THelper, uint64x2_t>();
+}
+
+void TSSEEmulTest::Test_mm_and_si128() {
+ struct THelper {
+ static ui64 Call(const ui64 op1, const ui64 op2) {
+ return op1 & op2;
+ }
+ };
+
+ Test_mm_dualop<ui64, 2, Wrap(_mm_and_si128), THelper, uint64x2_t>();
+}
+
+void TSSEEmulTest::Test_mm_andnot_si128() {
+ struct THelper {
+ static ui64 Call(const ui64 op1, const ui64 op2) {
+ return (~op1) & op2;
+ }
+ };
+
+ Test_mm_dualop<ui64, 2, Wrap(_mm_andnot_si128), THelper, uint64x2_t>();
+}
+
+template <typename TElem>
+struct THelperCMPEQ {
+ static TElem Call(const TElem op1, const TElem op2) {
+ return op1 == op2 ? ~TElem(0) : TElem(0);
+ }
+};
+
+void TSSEEmulTest::Test_mm_cmpeq_epi8() {
+ Test_mm_dualcmp<ui8, 16, Wrap(_mm_cmpeq_epi8),
+ THelperCMPEQ<ui8>, uint8x16_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmpeq_epi16() {
+ Test_mm_dualcmp<ui16, 8, Wrap(_mm_cmpeq_epi16),
+ THelperCMPEQ<ui16>, uint16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmpeq_epi32() {
+ Test_mm_dualcmp<ui32, 4, Wrap(_mm_cmpeq_epi32),
+ THelperCMPEQ<ui32>, uint32x4_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmpeq_ps() {
+ struct THelperFloat {
+ static float Call(const float op1, const float op2) {
+ union Cast {
+ unsigned int AsUInt;
+ float AsFloat;
+ };
+ Cast value;
+ value.AsUInt = op1 == op2 ? 0xFFFFFFFF : 0;
+ return value.AsFloat;
+ }
+ };
+
+ Test_mm_dualcmp<float, 4, WrapF(_mm_cmpeq_ps),
+ THelperFloat, float32x4_t, __m128>();
+}
+
+template <typename TElem>
+struct THelperCMPGT {
+ static TElem Call(const TElem op1, const TElem op2) {
+ return op1 > op2 ? ~TElem(0) : TElem(0);
+ }
+};
+
+void TSSEEmulTest::Test_mm_cmpgt_epi8() {
+ Test_mm_dualcmp<i8, 16, Wrap(_mm_cmpgt_epi8),
+ THelperCMPGT<i8>, int8x16_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmpgt_epi16() {
+ Test_mm_dualcmp<i16, 8, Wrap(_mm_cmpgt_epi16),
+ THelperCMPGT<i16>, int16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmpgt_epi32() {
+ Test_mm_dualcmp<i32, 4, Wrap(_mm_cmpgt_epi32),
+ THelperCMPGT<i32>, int32x4_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmpgt_ps() {
+ struct THelperFloat {
+ static float Call(const float op1, const float op2) {
+ union Cast {
+ unsigned int AsUInt;
+ float AsFloat;
+ };
+ Cast value;
+ value.AsUInt = op1 > op2 ? 0xFFFFFFFF : 0;
+ return value.AsFloat;
+ }
+ };
+
+ Test_mm_dualcmp<float, 4, WrapF(_mm_cmpgt_ps),
+ THelperFloat, float32x4_t, __m128>();
+}
+
+template <typename TElem>
+struct THelperCMPLT {
+ static TElem Call(const TElem op1, const TElem op2) {
+ return op1 < op2 ? ~TElem(0) : TElem(0);
+ }
+};
+
+void TSSEEmulTest::Test_mm_cmplt_epi8() {
+ Test_mm_dualcmp<i8, 16, Wrap(_mm_cmplt_epi8),
+ THelperCMPLT<i8>, int8x16_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmplt_epi16() {
+ Test_mm_dualcmp<i16, 8, Wrap(_mm_cmplt_epi16),
+ THelperCMPLT<i16>, int16x8_t>();
+}
+
+void TSSEEmulTest::Test_mm_cmplt_epi32() {
+ Test_mm_dualcmp<i32, 4, Wrap(_mm_cmplt_epi32),
+ THelperCMPLT<i32>, int32x4_t>();
+}
+
+template <typename TElem, int elemCount,
+ typename TFunc, typename TOp, typename TVectorType>
+void TSSEEmulTest::Test_mm_setter_epiXX() {
+ char data[64] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C',
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF',
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C',
+ '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
+ TElem* dataw = reinterpret_cast<TElem*>(&data);
+
+ for (unsigned dataItem = 0; dataItem < elemCount * 4; ++dataItem) {
+ TVectorType value = TFunc(dataw[dataItem]);
+
+ for (unsigned i = 0; i < elemCount; ++i)
+ UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<TOp>::As(value)[i]);
+ }
+}
+
+void TSSEEmulTest::Test_mm_set1_epi8() {
+ Test_mm_setter_epiXX<i8, 16, Wrap(_mm_set1_epi8), int8x16_t, __m128i>();
+}
+void TSSEEmulTest::Test_mm_set1_epi16() {
+ Test_mm_setter_epiXX<i16, 8, Wrap(_mm_set1_epi16), int16x8_t, __m128i>();
+}
+void TSSEEmulTest::Test_mm_set1_epi32() {
+ Test_mm_setter_epiXX<i32, 4, Wrap(_mm_set1_epi32), int32x4_t, __m128i>();
+}
+void TSSEEmulTest::Test_mm_set1_ps() {
+ Test_mm_setter_epiXX<float, 4, WrapF(_mm_set1_ps), float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_set_ps1() {
+ Test_mm_setter_epiXX<float, 4, WrapF(_mm_set_ps1), float32x4_t, __m128>();
+}
+
+void TSSEEmulTest::Test_mm_setzero_si128() {
+ __m128i value = _mm_setzero_si128();
+ for (unsigned i = 0; i < 4; ++i)
+ UNIT_ASSERT_EQUAL(0, TQType<uint32x4_t>::As(value)[i]);
+}
+
+void TSSEEmulTest::Test_mm_setzero_ps() {
+ __m128 value = _mm_setzero_ps();
+ for (unsigned i = 0; i < 4; ++i)
+ UNIT_ASSERT_EQUAL(0.0, TQType<float32x4_t>::As(value)[i]);
+}
+
+void TSSEEmulTest::Test_mm_setzero_pd() {
+ __m128d value = _mm_setzero_pd();
+ for (unsigned i = 0; i < 2; ++i)
+ UNIT_ASSERT_EQUAL(0.0, TQType<float64x2_t>::As(value)[i]);
+}
+
+void TSSEEmulTest::Test_mm_loadl_epi64() {
+ char data[64] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C',
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF',
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C',
+ '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
+ ui64* dataw = reinterpret_cast<ui64*>(&data);
+
+ for (unsigned dataItem = 0; dataItem < 8; ++dataItem) {
+ __m128i value = _mm_loadl_epi64((__m128i const*)&dataw[dataItem]);
+
+ UNIT_ASSERT_EQUAL(dataw[dataItem], TQType<uint64x2_t>::As(value)[0]);
+ UNIT_ASSERT_EQUAL(0, TQType<uint64x2_t>::As(value)[1]);
+ }
+}
+
+void TSSEEmulTest::Test_mm_storel_epi64() {
+ char data[64] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x00', '\x55', '\x77', '\x66', '\x1C',
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF',
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x00', '\x00', '\x00',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x66', '\x1C',
+ '\x99', '\x33', '\xFF', '\xCC', '\x88', '\x66', '\x77', '\x44',
+ '\x33', '\x99', '\x44', '\x88', '\xCC', '\xBB', '\x22', '\xFF'};
+ ui64* dataw = reinterpret_cast<ui64*>(&data);
+
+ for (unsigned dataItem = 0; dataItem < 4; ++dataItem) {
+ __m128i value = _mm_loadu_si128((__m128i*)&dataw[dataItem * 2]);
+
+ ui64 buf[2] = {55, 81};
+ _mm_storel_epi64((__m128i*)&buf, value);
+
+ UNIT_ASSERT_EQUAL(dataw[dataItem * 2], buf[0]);
+ UNIT_ASSERT_EQUAL(81, buf[1]);
+ }
+}
+
+void TSSEEmulTest::Test_mm_shuffle_epi32() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ ui32* dataw = reinterpret_cast<ui32*>(&data);
+ __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ int coding[4] = {1, 3, 0, 2};
+ __m128i result = _mm_shuffle_epi32(value, _MM_SHUFFLE(2, 0, 3, 1));
+
+ for (unsigned i = 0; i < 4; ++i)
+ UNIT_ASSERT_EQUAL(dataw[coding[i]],
+ TQType<uint32x4_t>::As(result)[i]);
+}
+
+static int GetHighBitAt(char data, int at) {
+ ui8 udata = data & 0x80;
+ return int(udata >> 7) << at;
+}
+
+void TSSEEmulTest::Test_mm_movemask_epi8() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ int result = _mm_movemask_epi8(value);
+ int verify = 0;
+ for (unsigned i = 0; i < 16; ++i) {
+ verify |= GetHighBitAt(data[i], i);
+ }
+
+ UNIT_ASSERT_EQUAL(result, verify);
+}
+
+void TSSEEmulTest::Test_mm_movemask_ps() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ __m128 value = _mm_loadu_ps((float*)&data);
+
+ int result = _mm_movemask_ps(value);
+ int verify = 0;
+ for (unsigned i = 0; i < 4; ++i) {
+ verify |= GetHighBitAt(data[i * 4 + 3], i);
+ }
+
+ UNIT_ASSERT_EQUAL(result, verify);
+}
+
+void TSSEEmulTest::Test_mm_movemask_ps_2() {
+ char data[16] = {
+ '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF',
+ '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF'};
+ __m128 value = _mm_loadu_ps((float*)&data);
+
+ int result = _mm_movemask_ps(value);
+ UNIT_ASSERT_EQUAL(result, 0xf);
+}
+
+void TSSEEmulTest::Test_mm_cvtsi128_si32() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ int result = _mm_cvtsi128_si32(value);
+ i32* datap = reinterpret_cast<i32*>(&data);
+ int verify = datap[0];
+
+ UNIT_ASSERT_EQUAL(result, verify);
+}
+
+void TSSEEmulTest::Test_mm_cvtsi128_si64() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ i64 result = _mm_cvtsi128_si64(value);
+ i64* datap = reinterpret_cast<i64*>(&data);
+ i64 verify = datap[0];
+
+ UNIT_ASSERT_EQUAL(result, verify);
+}
+
+void TSSEEmulTest::Test_mm_set_epi16() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ i16* dataw = reinterpret_cast<i16*>(&data);
+ ui64* dataq = reinterpret_cast<ui64*>(&data);
+
+ __m128i result = _mm_set_epi16(dataw[7], dataw[6], dataw[5], dataw[4],
+ dataw[3], dataw[2], dataw[1], dataw[0]);
+ ui64 buf[2] = {53, 81};
+ _mm_storeu_si128((__m128i*)&buf, result);
+
+ UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
+ UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
+}
+
+void TSSEEmulTest::Test_mm_set_epi32() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ i32* dataw = reinterpret_cast<i32*>(&data);
+ ui64* dataq = reinterpret_cast<ui64*>(&data);
+
+ __m128i result = _mm_set_epi32(dataw[3], dataw[2], dataw[1], dataw[0]);
+ ui64 buf[2] = {53, 81};
+ _mm_storeu_si128((__m128i*)&buf, result);
+
+ UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
+ UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
+}
+
+void TSSEEmulTest::Test_mm_set_ps() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ float* dataw = reinterpret_cast<float*>(&data);
+ ui64* dataq = reinterpret_cast<ui64*>(&data);
+
+ __m128 result = _mm_set_ps(dataw[3], dataw[2], dataw[1], dataw[0]);
+ ui64 buf[2] = {53, 81};
+ _mm_storeu_ps((float*)&buf, result);
+
+ UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
+ UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
+}
+
+void TSSEEmulTest::Test_mm_set_pd() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ double* dataw = reinterpret_cast<double*>(&data);
+ ui64* dataq = reinterpret_cast<ui64*>(&data);
+
+ __m128d result = _mm_set_pd(dataw[1], dataw[0]);
+ ui64 buf[2] = {53, 81};
+ _mm_storeu_pd((double*)&buf, result);
+
+ UNIT_ASSERT_EQUAL(buf[0], dataq[0]);
+ UNIT_ASSERT_EQUAL(buf[1], dataq[1]);
+}
+
+void TSSEEmulTest::Test_mm_cvtsi32_si128() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ i32* dataw = reinterpret_cast<i32*>(&data);
+
+ __m128i result = _mm_cvtsi32_si128(dataw[0]);
+ i32 buf[4] = {53, 81, -43, 2132};
+ _mm_storeu_si128((__m128i*)&buf, result);
+
+ UNIT_ASSERT_EQUAL(buf[0], dataw[0]);
+ UNIT_ASSERT_EQUAL(buf[1], 0);
+ UNIT_ASSERT_EQUAL(buf[2], 0);
+ UNIT_ASSERT_EQUAL(buf[3], 0);
+}
+
+void TSSEEmulTest::Test_mm_cvtsi64_si128() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ i64* dataw = reinterpret_cast<i64*>(&data);
+
+ __m128i result = _mm_cvtsi64_si128(dataw[0]);
+ i64 buf[2] = {7, 8};
+ _mm_storeu_si128((__m128i*)&buf, result);
+
+ UNIT_ASSERT_EQUAL(buf[0], dataw[0]);
+ UNIT_ASSERT_EQUAL(buf[1], 0);
+}
+
+template <typename TElem, typename TNarrow, unsigned elemCount, typename TFunc>
+void TSSEEmulTest::Test_mm_packs_epiXX() {
+ char data[32] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x00', '\x66', '\x1C',
+ '\x99', '\x33', '\x1C', '\x55', '\x00', '\x00', '\x00', '\x00',
+ '\x00', '\xAA', '\x00', '\x00', '\xCC', '\xBB', '\x22', '\xFF'};
+ __m128i value0 = _mm_loadu_si128((__m128i*)&data);
+ __m128i value1 = _mm_loadu_si128(((__m128i*)&data) + 1);
+ TElem* dataw = reinterpret_cast<TElem*>(&data);
+
+ __m128i result = TFunc(value0, value1);
+
+ TNarrow verify[elemCount];
+ for (unsigned i = 0; i < elemCount; ++i) {
+ TElem sum = dataw[i];
+ if (sum > std::numeric_limits<TNarrow>::max())
+ sum = std::numeric_limits<TNarrow>::max();
+ if (sum < std::numeric_limits<TNarrow>::min())
+ sum = std::numeric_limits<TNarrow>::min();
+ verify[i] = TNarrow(sum);
+ }
+
+ ui64* verifyp = (ui64*)&verify;
+ UNIT_ASSERT_EQUAL(verifyp[0], TQType<uint64x2_t>::As(result)[0]);
+ UNIT_ASSERT_EQUAL(verifyp[1], TQType<uint64x2_t>::As(result)[1]);
+}
+
+void TSSEEmulTest::Test_mm_packs_epi16() {
+ Test_mm_packs_epiXX<i16, i8, 16, Wrap(_mm_packs_epi16)>();
+}
+void TSSEEmulTest::Test_mm_packs_epi32() {
+ Test_mm_packs_epiXX<i32, i16, 8, Wrap(_mm_packs_epi32)>();
+}
+void TSSEEmulTest::Test_mm_packus_epi16() {
+ Test_mm_packs_epiXX<i16, ui8, 16, Wrap(_mm_packus_epi16)>();
+}
+
+void TSSEEmulTest::Test_mm_extract_epi8() {
+ alignas(16) char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ const ui8* dataw = reinterpret_cast<const ui8*>(&data);
+ const __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 1)), int(dataw[1]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 2)), int(dataw[2]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 3)), int(dataw[3]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 4)), int(dataw[4]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 5)), int(dataw[5]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 6)), int(dataw[6]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 7)), int(dataw[7]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 8)), int(dataw[8]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 9)), int(dataw[9]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 10)), int(dataw[10]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 11)), int(dataw[11]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 12)), int(dataw[12]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 13)), int(dataw[13]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 14)), int(dataw[14]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi8(value, 15)), int(dataw[15]));
+}
+
+void TSSEEmulTest::Test_mm_extract_epi16() {
+ alignas(16) char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ const ui16* dataw = reinterpret_cast<const ui16*>(&data);
+ const __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 0)), int(dataw[0]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 1)), int(dataw[1]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 2)), int(dataw[2]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 3)), int(dataw[3]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 4)), int(dataw[4]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 5)), int(dataw[5]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 6)), int(dataw[6]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi16(value, 7)), int(dataw[7]));
+}
+
+void TSSEEmulTest::Test_mm_extract_epi64() {
+ alignas(16) char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ const ui64* dataw = reinterpret_cast<const ui64*>(&data);
+ const __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 0)), (long long)(dataw[0]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi64(value, 1)), (long long)(dataw[1]));
+}
+
+void TSSEEmulTest::Test_mm_extract_epi32() {
+ alignas(16) char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ const ui32* dataw = reinterpret_cast<const ui32*>(&data);
+ const __m128i value = _mm_loadu_si128((__m128i*)&data);
+
+ UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 0)), int(dataw[0]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 1)), int(dataw[1]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 2)), int(dataw[2]));
+ UNIT_ASSERT_EQUAL((_mm_extract_epi32(value, 3)), int(dataw[3]));
+}
+
+void TSSEEmulTest::Test_MM_TRANSPOSE4_PS() {
+ char data0[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ char data1[16] = {
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
+ char data2[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ char data3[16] = {
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
+
+ __m128 value0 = _mm_loadu_ps((float*)&data0);
+ __m128 value1 = _mm_loadu_ps((float*)&data1);
+ __m128 value2 = _mm_loadu_ps((float*)&data2);
+ __m128 value3 = _mm_loadu_ps((float*)&data3);
+
+ _MM_TRANSPOSE4_PS(value0, value1, value2, value3);
+
+ ui64 tbuf0[2] = {0, 0};
+ ui64 tbuf1[2] = {0, 0};
+ ui64 tbuf2[2] = {0, 0};
+ ui64 tbuf3[2] = {0, 0};
+
+ _mm_storeu_ps((float*)&tbuf0, value0);
+ _mm_storeu_ps((float*)&tbuf1, value1);
+ _mm_storeu_ps((float*)&tbuf2, value2);
+ _mm_storeu_ps((float*)&tbuf3, value3);
+
+ char tdata0[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55',
+ '\xAA', '\x00', '\xFF', '\xCC', '\x99', '\x33', '\x1C', '\x55'};
+ char tdata1[16] = {
+ '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44',
+ '\x11', '\x22', '\xBB', '\xAA', '\x88', '\x66', '\x77', '\x44'};
+ char tdata2[16] = {
+ '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11',
+ '\x33', '\x99', '\x44', '\x88', '\x00', '\xAA', '\xAA', '\x11'};
+ char tdata3[16] = {
+ '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF',
+ '\x55', '\x77', '\x66', '\x1C', '\xCC', '\xBB', '\x22', '\xFF'};
+
+ UNIT_ASSERT(memcmp(tbuf0, tdata0, 16) == 0);
+ UNIT_ASSERT(memcmp(tbuf1, tdata1, 16) == 0);
+ UNIT_ASSERT(memcmp(tbuf2, tdata2, 16) == 0);
+ UNIT_ASSERT(memcmp(tbuf3, tdata3, 16) == 0);
+}
+
+template <typename TFrom, typename TTo, unsigned elemCount,
+ typename TLoadVector, typename TResultVector,
+ typename TElemFunc, typename TFunc, typename TOp>
+void TSSEEmulTest::Test_mm_convertop() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ TFrom* datap = reinterpret_cast<TFrom*>(&data);
+
+ TLoadVector value = TFuncLoad<TLoadVector>(&data);
+
+ TTo procData[elemCount];
+ for (unsigned i = 0; i < elemCount; ++i) {
+ procData[i] = TElemFunc::Call(datap[i]);
+ }
+
+ TResultVector result = TFunc(value);
+
+ for (unsigned i = 0; i < elemCount; ++i) {
+ UNIT_ASSERT_EQUAL(procData[i], TQType<TOp>::As(result)[i]);
+ }
+}
+
+void TSSEEmulTest::Test_mm_cvtepi32_ps() {
+ struct THelper {
+ static float Call(const i32 op) {
+ return float(op);
+ }
+ };
+ Test_mm_convertop<i32, float, 4, __m128i, __m128,
+ THelper, WrapF(_mm_cvtepi32_ps), float32x4_t>();
+};
+
+void TSSEEmulTest::Test_mm_cvtps_epi32() {
+ struct THelper {
+ static i32 Call(const float op) {
+ return i32(op);
+ }
+ };
+ Test_mm_convertop<float, i32, 4, __m128, __m128i,
+ THelper, T_mm_CallWrapper<__m128i, decltype(_mm_cvtps_epi32), _mm_cvtps_epi32>, int32x4_t>();
+};
+
+void TSSEEmulTest::Test_mm_cvttps_epi32() {
+ struct THelper {
+ static i32 Call(const float op) {
+ return i32(op);
+ }
+ };
+ Test_mm_convertop<float, i32, 4, __m128, __m128i,
+ THelper, Wrap(_mm_cvttps_epi32), int32x4_t>();
+};
+
+template <typename TLoadVector, typename TCastVector,
+ typename TFunc, TFunc* func>
+void TSSEEmulTest::Test_mm_castXX() {
+ char data[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+
+ TLoadVector value = TFuncLoad<TLoadVector>(&data);
+ const TLoadVector constvalue = TFuncLoad<TLoadVector>(&data);
+ TCastVector casted = func(value);
+ const TCastVector constcasted = func(constvalue);
+ char verify[16];
+ char constverify[16];
+ TFuncStore<TCastVector>(&verify, casted);
+ TFuncStore<TCastVector>(&constverify, constcasted);
+
+ UNIT_ASSERT(memcmp(&data, &verify, 16) == 0);
+ UNIT_ASSERT(memcmp(&data, &constverify, 16) == 0);
+};
+
+void TSSEEmulTest::Test_mm_castsi128_ps() {
+ Test_mm_castXX<__m128i, __m128,
+ decltype(_mm_castsi128_ps), _mm_castsi128_ps>();
+}
+
+void TSSEEmulTest::Test_mm_castps_si128() {
+ Test_mm_castXX<__m128, __m128i,
+ decltype(_mm_castps_si128), _mm_castps_si128>();
+}
+
+void TSSEEmulTest::Test_mm_mul_epu32() {
+ char data0[16] = {
+ '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA',
+ '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C'};
+ char data1[16] = {
+ '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44',
+ '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF'};
+ ui32* dataw0 = reinterpret_cast<ui32*>(&data0);
+ ui32* dataw1 = reinterpret_cast<ui32*>(&data1);
+
+ __m128i value0 = _mm_loadu_si128((__m128i*)&data0);
+ __m128i value1 = _mm_loadu_si128((__m128i*)&data1);
+
+ ui64 mul0 = (ui64) dataw0[0] * (ui64) dataw1[0];
+ ui64 mul1 = (ui64) dataw0[2] * (ui64) dataw1[2];
+
+ __m128i result = _mm_mul_epu32(value0, value1);
+
+ UNIT_ASSERT_EQUAL(mul0, TQType<uint64x2_t>::As(result)[0]);
+ UNIT_ASSERT_EQUAL(mul1, TQType<uint64x2_t>::As(result)[1]);
+}
+
+void TSSEEmulTest::Test_mm_cmpunord_ps() {
+ alignas(16) float valuesBits[4] = {1.f, 2.f, 3.f, 4.f};
+ alignas(16) float values2Bits[4] = {5.f, 6.f, 7.f, 8.f};
+
+ alignas(16) char allfs[16] = {
+ '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff',
+ '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff'
+ };
+
+ alignas(16) char allzeroes[16] = {
+ '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00',
+ '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'
+ };
+
+ const __m128 qnan = _mm_set_ps1(std::numeric_limits<float>::quiet_NaN());
+ const __m128 snan = _mm_set_ps1(std::numeric_limits<float>::signaling_NaN());
+ const __m128 values = _mm_loadu_ps((const float*) valuesBits);
+ const __m128 values2 = _mm_loadu_ps((const float*) values2Bits);
+
+ const __m128 mask1 = _mm_cmpunord_ps(qnan, qnan);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask1, &allfs, sizeof(allfs)), 0);
+
+ const __m128 mask2 = _mm_cmpunord_ps(values, values);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask2, &allzeroes, sizeof(allzeroes)), 0);
+
+ const __m128 mask3 = _mm_cmpunord_ps(snan, snan);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask3, &allfs, sizeof(allfs)), 0);
+
+ const __m128 mask4 = _mm_cmpunord_ps(qnan, values);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask4, &allfs, sizeof(allfs)), 0);
+
+ const __m128 mask5 = _mm_cmpunord_ps(snan, values);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask5, &allfs, sizeof(allfs)), 0);
+
+ const __m128 mask6 = _mm_cmpunord_ps(qnan, snan);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask6, &allfs, sizeof(allfs)), 0);
+
+ const __m128 mask7 = _mm_cmpunord_ps(values, values2);
+ UNIT_ASSERT_EQUAL(::memcmp(&mask7, &allzeroes, sizeof(allzeroes)), 0);
+}
+
+void TSSEEmulTest::Test_mm_store_ss() {
+ alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f};
+ const __m128 value = _mm_loadu_ps(valueBits);
+ float res = std::numeric_limits<float>::signaling_NaN();
+ _mm_store_ss(&res, value);
+ UNIT_ASSERT_EQUAL(res, 1.f);
+}
+
+void TSSEEmulTest::Test_mm_store_ps() {
+ alignas(16) const float valueBits[4] = {1.f, 2.f, 3.f, 4.f};
+ const __m128 value = _mm_loadu_ps(valueBits);
+ float res[4] = {0.f};
+ _mm_storeu_ps(res, value);
+ UNIT_ASSERT_EQUAL(res[0], 1.f);
+ UNIT_ASSERT_EQUAL(res[1], 2.f);
+ UNIT_ASSERT_EQUAL(res[2], 3.f);
+ UNIT_ASSERT_EQUAL(res[3], 4.f);
+}
+
+void TSSEEmulTest::Test_mm_storeu_pd() {
+ alignas(16) const double valueBits[4] = {1., 2., 3., 4.};
+ for (size_t i = 0; i != 3; ++i) {
+ const __m128d value = _mm_loadu_pd(&valueBits[i]);
+ alignas(16) double res[4];
+ for (size_t shift = 0; shift != 3; ++shift) {
+ _mm_storeu_pd(&res[shift], value);
+ for (size_t j = 0; j != 2; ++j) {
+ UNIT_ASSERT_EQUAL_C(res[j + shift], valueBits[i + j], "res: " << HexEncode(&res[shift], 16) << " vs etalon: " << HexEncode(&valueBits[i], 16));
+ }
+ }
+ }
+}
+
+void TSSEEmulTest::Test_mm_andnot_ps() {
+ alignas(16) const char firstBits[16] = {
+ '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff',
+ '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff'
+ };
+
+ alignas(16) const char secondBits[16] = {
+ '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff',
+ '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff'
+ };
+
+ alignas(16) const char resBits[16] = {
+ '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00',
+ '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00'
+ };
+
+ const __m128 value1 = _mm_loadu_ps((const float*) firstBits);
+ const __m128 value2 = _mm_loadu_ps((const float*) secondBits);
+ const __m128 res = _mm_andnot_ps(value1, value2);
+
+ UNIT_ASSERT_EQUAL(::memcmp(&res, resBits, sizeof(resBits)), 0);
+}
+
+void TSSEEmulTest::Test_mm_shuffle_ps() {
+ alignas(16) const float first[4] = {1.f, 2.f, 3.f, 4.f};
+ alignas(16) const float second[4] = {5.f, 6.f, 7.f, 8.f};
+ alignas(16) const float etalon[4] = {3.f, 4.f, 5.f, 6.f};
+
+ const __m128 value1 = _mm_loadu_ps(first);
+ const __m128 value2 = _mm_loadu_ps(second);
+ const __m128 res = _mm_shuffle_ps(value1, value2, _MM_SHUFFLE(1, 0, 3, 2));
+
+ UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0);
+}
+
+void TSSEEmulTest::Test_mm_shuffle_pd() {
+ const double first[2] = {1.3, 2.3};
+ const double second[2] = {5.3, 6.3};
+ const double etalon0[2] = {1.3, 5.3};
+ const double etalon1[2] = {2.3, 5.3};
+ const double etalon2[2] = {1.3, 6.3};
+ const double etalon3[2] = {2.3, 6.3};
+
+ const __m128d value1 = _mm_loadu_pd(first);
+ const __m128d value2 = _mm_loadu_pd(second);
+
+ __m128d res = _mm_shuffle_pd(value1, value2, 0);
+ UNIT_ASSERT_EQUAL(::memcmp(&res, etalon0, sizeof(etalon0)), 0);
+
+ res = _mm_shuffle_pd(value1, value2, 1);
+ UNIT_ASSERT_EQUAL(::memcmp(&res, etalon1, sizeof(etalon1)), 0);
+
+ res = _mm_shuffle_pd(value1, value2, 2);
+ UNIT_ASSERT_EQUAL(::memcmp(&res, etalon2, sizeof(etalon2)), 0);
+
+ res = _mm_shuffle_pd(value1, value2, 3);
+ UNIT_ASSERT_EQUAL(::memcmp(&res, etalon3, sizeof(etalon3)), 0);
+}
+
+void TSSEEmulTest::Test_mm_cvtsd_f64() {
+ const double first[2] = {1.3, 2.3};
+ const double second[2] = {5.3, 6.3};
+
+ const __m128d value1 = _mm_loadu_pd(first);
+ const __m128d value2 = _mm_loadu_pd(second);
+
+ UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value1), 1.3);
+ UNIT_ASSERT_EQUAL(_mm_cvtsd_f64(value2), 5.3);
+}
+
+void TSSEEmulTest::Test_mm_loadl_pd() {
+ const double first[2] = {1.3, 2.3};
+ const double second[2] = {5.3, 6.3};
+ const double firstEtalon[2] = {10.13, 2.3};
+ const double secondEtalon[2] = {11.13, 6.3};
+
+ double newFirst = 10.13;
+ double newSecond = 11.13;
+
+ __m128d value1 = _mm_loadu_pd(first);
+ __m128d value2 = _mm_loadu_pd(second);
+ value1 = _mm_loadl_pd(value1, &newFirst);
+ value2 = _mm_loadl_pd(value2, &newSecond);
+ UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0);
+ UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0);
+}
+
+void TSSEEmulTest::Test_mm_loadh_pd() {
+ const double first[2] = {1.3, 2.3};
+ const double second[2] = {5.3, 6.3};
+ const double firstEtalon[2] = {1.3, 10.13};
+ const double secondEtalon[2] = {5.3, 11.13};
+
+ double newFirst = 10.13;
+ double newSecond = 11.13;
+
+ __m128d value1 = _mm_loadu_pd(first);
+ __m128d value2 = _mm_loadu_pd(second);
+ value1 = _mm_loadh_pd(value1, &newFirst);
+ value2 = _mm_loadh_pd(value2, &newSecond);
+ UNIT_ASSERT_EQUAL(::memcmp(&value1, firstEtalon, sizeof(firstEtalon)), 0);
+ UNIT_ASSERT_EQUAL(::memcmp(&value2, secondEtalon, sizeof(secondEtalon)), 0);
+}
+
+void TSSEEmulTest::Test_mm_or_ps() {
+ alignas(16) const char bytes1[16] = {
+ '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff',
+ '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff'
+ };
+
+ alignas(16) const char bytes2[16] = {
+ '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff',
+ '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff'
+ };
+
+ alignas(16) const char etalon[16] = {
+ '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff',
+ '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff'
+ };
+
+ const __m128 value1 = _mm_loadu_ps((const float*) bytes1);
+ const __m128 value2 = _mm_loadu_ps((const float*) bytes2);
+ const __m128 res = _mm_or_ps(value1, value2);
+
+ UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0);
+}
+
+void TSSEEmulTest::Test_mm_loadu_pd() {
+ alignas(16) double stub[4] = {
+ 0.f, 1.f,
+ 2.f, 3.f
+ };
+
+ for (size_t shift = 0; shift != 3; ++shift) {
+ const __m128d val = _mm_loadu_pd(&stub[shift]);
+ alignas(16) double res[2];
+ _mm_store_pd(res, val);
+
+ for (size_t i = 0; i != 2; ++i) {
+ UNIT_ASSERT_EQUAL_C(res[i], stub[shift + i], "res: " << HexEncode(res, 16) << " vs etalon: " << HexEncode(&stub[shift], 16));
+ }
+ }
+}
+
+void TSSEEmulTest::Test_mm_rsqrt_ps() {
+ alignas(16) const char bytes[16] = {
+ '\x00', '\x00', '\x28', '\x42', // 42.f
+ '\x00', '\x98', '\x84', '\x45', // 4243.f
+ '\x60', '\x26', '\xcf', '\x48', // 424243.f
+ '\xed', '\xd5', '\x21', '\x4c' // 42424243.f
+ };
+ const __m128 value = _mm_loadu_ps((const float*)bytes);
+ const __m128 result = _mm_rsqrt_ps(value);
+ alignas(16) float res[4];
+ _mm_store_ps(res, result);
+ float fResult = 0.f;
+ for (size_t i = 0; i < 4; ++i) {
+ memcpy(&fResult, &bytes[i * 4], 4);
+ fResult = 1.f / std::sqrt(fResult);
+ UNIT_ASSERT_DOUBLES_EQUAL_C(res[i], fResult, 1e-3, "res: " << fResult << " vs etalon " << res[i]);
+ }
+}
+
+namespace NHelpers {
+
+ static __m128i Y_FORCE_INLINE GetCmp16(const __m128 &c0, const __m128 &c1, const __m128 &c2, const __m128 &c3, const __m128 test) {
+ const __m128i r0 = _mm_castps_si128(_mm_cmpgt_ps(c0, test));
+ const __m128i r1 = _mm_castps_si128(_mm_cmpgt_ps(c1, test));
+ const __m128i r2 = _mm_castps_si128(_mm_cmpgt_ps(c2, test));
+ const __m128i r3 = _mm_castps_si128(_mm_cmpgt_ps(c3, test));
+ const __m128i packed = _mm_packs_epi16(_mm_packs_epi32(r0, r1), _mm_packs_epi32(r2, r3));
+ return _mm_and_si128(_mm_set1_epi8(0x01), packed);
+ }
+
+ static __m128i Y_FORCE_INLINE GetCmp16(const float *factors, const __m128 test) {
+ const __m128 *ptr = (__m128 *)factors;
+ return GetCmp16(ptr[0], ptr[1], ptr[2], ptr[3], test);
+ }
+
+ template<size_t Num>
+ void DoLane(size_t length, const float *factors, ui32 *& dst, const float *&values) {
+ for (size_t i = 0; i < length; ++i) {
+ __m128 value = _mm_set1_ps(values[i]);
+ __m128i agg = GetCmp16(factors, value);
+ if (Num > 1) {
+ agg = _mm_add_epi16(agg, _mm_slli_epi16(GetCmp16(&factors[64], value), 1));
+ }
+ _mm_store_si128((__m128i *)&dst[4 * i], agg);
+ }
+ }
+}
+
+void TSSEEmulTest::Test_matrixnet_powerpc() {
+ static constexpr size_t length = 10;
+ alignas(16) float factors[1024];
+ alignas(16) ui32 valP[4 * length] = { 0 };
+ float values[length];
+ TReallyFastRng32 rng(42);
+ for (size_t i = 0; i < 1024; ++i) {
+ factors[i] = rng.GenRandReal2();
+ }
+ for (size_t i = 0; i < length; ++i) {
+ values[i] = rng.GenRandReal2();
+ }
+ ui32* val = reinterpret_cast<ui32*>(valP);
+ const float* vals = reinterpret_cast<const float*>(values);
+ NHelpers::DoLane<2>(length, factors, val, vals);
+ static const ui32 etalon[4 * length] = {
+ 2, 33554432, 258, 33554433, 50529027,
+ 50529027, 50529027, 50529027, 50528770,
+ 33685763, 33555203, 50462723, 50528770,
+ 33685763, 33555203, 50462723, 50529026,
+ 33751299, 50529027, 50463491, 2, 33554432,
+ 258, 33554433, 50397698, 33685761, 259,
+ 50462721, 50332162, 33554689, 259, 50462721,
+ 50528770, 33685761, 33555203, 50462723,
+ 50529026, 33685763, 50463491, 50463235
+ };
+ for (size_t i = 0; i < 4 * length; ++i) {
+ UNIT_ASSERT_EQUAL(valP[i], etalon[i]);
+ }
+}
diff --git a/library/cpp/sse/ut/ya.make b/library/cpp/sse/ut/ya.make
new file mode 100644
index 0000000000..45e104971e
--- /dev/null
+++ b/library/cpp/sse/ut/ya.make
@@ -0,0 +1,13 @@
+UNITTEST_FOR(library/cpp/sse)
+
+OWNER(danlark)
+
+SRCS(
+ test.cpp
+)
+
+IF (ARCH_X86_64)
+ CFLAGS(-msse4.1 -msse4.2)
+ENDIF()
+
+END()
diff --git a/library/cpp/sse/ya.make b/library/cpp/sse/ya.make
new file mode 100644
index 0000000000..d2351e791d
--- /dev/null
+++ b/library/cpp/sse/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+OWNER(
+ g:base
+ danlark
+)
+
+SRCS(
+ sse.cpp
+)
+
+END()