1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
#include "dot_product_vnni.h"
#include "dot_product_avx2.h"
#include <util/system/compiler.h>
#include <util/system/platform.h>
#if defined(__AVX512VNNI__) && defined(__AVX512BW__) && defined(__AVX512F__)
#include <immintrin.h>
namespace {
Y_FORCE_INLINE __m512i Load512i(const void* ptr) {
return _mm512_loadu_si512(ptr);
}
Y_FORCE_INLINE __m512i CorrectSignedLhsDotProduct(
const __m512i biasedDotProduct,
const __m512i rhsSum) noexcept
{
return _mm512_sub_epi32(biasedDotProduct, _mm512_slli_epi32(rhsSum, 7));
}
Y_FORCE_INLINE void AccumulateDotProductI8Vnni(
__m512i& dotProduct,
const i8* lhs,
const i8* rhs,
const __m512i zero,
const __m512i signBit,
const __m512i ones) noexcept
{
const __m512i l = Load512i(lhs);
const __m512i r = Load512i(rhs);
const __m512i unsignedL = _mm512_xor_si512(l, signBit);
// AVX512 VNNI has u8*i8 dot product. For signed lhs we compute
// (lhs + 128) * rhs and immediately subtract 128 * sum(rhs).
// Correcting each 64-byte chunk keeps the accumulator in the same
// range as a regular signed i8 dot product accumulator instead of
// accumulating a much larger biased intermediate value.
const __m512i biasedDotProduct = _mm512_dpbusd_epi32(zero, unsignedL, r);
const __m512i rhsSum = _mm512_dpbusd_epi32(zero, ones, r);
dotProduct = _mm512_add_epi32(dotProduct, CorrectSignedLhsDotProduct(biasedDotProduct, rhsSum));
}
Y_FORCE_INLINE void AccumulateTailDotProductI8Vnni(
__m512i& dotProduct,
const i8* lhs,
const i8* rhs,
const size_t length,
const __m512i zero,
const __m512i signBit,
const __m512i ones) noexcept
{
const __mmask64 mask = (ui64(1) << length) - 1;
const __m512i l = _mm512_maskz_loadu_epi8(mask, lhs);
const __m512i r = _mm512_maskz_loadu_epi8(mask, rhs);
const __m512i unsignedL = _mm512_xor_si512(l, signBit);
const __m512i biasedDotProduct = _mm512_dpbusd_epi32(zero, unsignedL, r);
const __m512i rhsSum = _mm512_dpbusd_epi32(zero, ones, r);
dotProduct = _mm512_add_epi32(dotProduct, CorrectSignedLhsDotProduct(biasedDotProduct, rhsSum));
}
i32 HsumI32(const __m512i v) noexcept {
__m128i sum = _mm512_castsi512_si128(v);
sum = _mm_add_epi32(sum, _mm512_extracti32x4_epi32(v, 1));
sum = _mm_add_epi32(sum, _mm512_extracti32x4_epi32(v, 2));
sum = _mm_add_epi32(sum, _mm512_extracti32x4_epi32(v, 3));
const __m128i hi64 = _mm_unpackhi_epi64(sum, sum);
const __m128i sum64 = _mm_add_epi32(hi64, sum);
const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
const __m128i sum32 = _mm_add_epi32(sum64, hi32);
return _mm_cvtsi128_si32(sum32);
}
}
i32 DotProductVnni(const i8* lhs, const i8* rhs, size_t length) noexcept {
if (length < 64) {
return DotProductAvx2(lhs, rhs, length);
}
const __m512i zero = _mm512_setzero_si512();
const __m512i signBit = _mm512_set1_epi8(-128);
const __m512i ones = _mm512_set1_epi8(1);
__m512i dotProduct0 = zero;
__m512i dotProduct1 = zero;
__m512i dotProduct2 = zero;
__m512i dotProduct3 = zero;
while (length >= 256) {
AccumulateDotProductI8Vnni(dotProduct0, lhs, rhs, zero, signBit, ones);
AccumulateDotProductI8Vnni(dotProduct1, lhs + 64, rhs + 64, zero, signBit, ones);
AccumulateDotProductI8Vnni(dotProduct2, lhs + 128, rhs + 128, zero, signBit, ones);
AccumulateDotProductI8Vnni(dotProduct3, lhs + 192, rhs + 192, zero, signBit, ones);
lhs += 256;
rhs += 256;
length -= 256;
}
while (length >= 64) {
AccumulateDotProductI8Vnni(dotProduct0, lhs, rhs, zero, signBit, ones);
lhs += 64;
rhs += 64;
length -= 64;
}
if (length > 0) {
AccumulateTailDotProductI8Vnni(dotProduct0, lhs, rhs, length, zero, signBit, ones);
}
const __m512i dotProduct = _mm512_add_epi32(
_mm512_add_epi32(dotProduct0, dotProduct1),
_mm512_add_epi32(dotProduct2, dotProduct3));
return HsumI32(dotProduct);
}
#else
i32 DotProductVnni(const i8* lhs, const i8* rhs, size_t length) noexcept {
return DotProductAvx2(lhs, rhs, length);
}
#endif
|
