summaryrefslogtreecommitdiffstats
path: root/library/cpp/dot_product
diff options
context:
space:
mode:
Diffstat (limited to 'library/cpp/dot_product')
-rw-r--r--library/cpp/dot_product/dot_product.cpp8
-rw-r--r--library/cpp/dot_product/dot_product_ut.cpp35
-rw-r--r--library/cpp/dot_product/dot_product_vnni.cpp125
-rw-r--r--library/cpp/dot_product/dot_product_vnni.h9
-rw-r--r--library/cpp/dot_product/ya.make2
5 files changed, 176 insertions, 3 deletions
diff --git a/library/cpp/dot_product/dot_product.cpp b/library/cpp/dot_product/dot_product.cpp
index 003dbe88335..d3f2b31b2d4 100644
--- a/library/cpp/dot_product/dot_product.cpp
+++ b/library/cpp/dot_product/dot_product.cpp
@@ -1,6 +1,7 @@
#include "dot_product.h"
#include "dot_product_sse.h"
#include "dot_product_avx2.h"
+#include "dot_product_vnni.h"
#include "dot_product_simple.h"
#include <library/cpp/sse/sse.h>
@@ -31,7 +32,6 @@ namespace NDotProductImpl {
namespace {
[[maybe_unused]] const int _ = [] {
if (!FromYaTest() && GetEnv("Y_NO_AVX_IN_DOT_PRODUCT") == "" && NX86::HaveAVX2() && NX86::HaveFMA()) {
- DotProductI8Impl = &DotProductAvx2;
DotProductUi8Impl = &DotProductAvx2;
DotProductI32Impl = &DotProductAvx2;
DotProductFloatImpl = &DotProductAvx2;
@@ -40,6 +40,12 @@ namespace NDotProductImpl {
TriWayDotProductImpl = &TriWayDotProductAvx2;
TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Avx2;
TriWayDotProductI8Impl = &TriWayDotProductI8Avx2;
+
+ if (GetEnv("Y_NO_VNNI_IN_DOT_PRODUCT") == "" && NX86::HaveAVX512VNNI() && NX86::HaveAVX512BW()) {
+ DotProductI8Impl = &DotProductVnni;
+ } else {
+ DotProductI8Impl = &DotProductAvx2;
+ }
} else {
#ifdef ARCADIA_SSE
DotProductI8Impl = &DotProductSse;
diff --git a/library/cpp/dot_product/dot_product_ut.cpp b/library/cpp/dot_product/dot_product_ut.cpp
index 6d07d1a3a31..8ba451f4964 100644
--- a/library/cpp/dot_product/dot_product_ut.cpp
+++ b/library/cpp/dot_product/dot_product_ut.cpp
@@ -1,6 +1,7 @@
#include "dot_product.h"
#include "dot_product_simple.h"
#include "dot_product_avx2.h"
+#include "dot_product_vnni.h"
#include <library/cpp/testing/unittest/registar.h>
@@ -48,16 +49,46 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) {
return sum;
}
+ bool HaveVnniDotProduct() {
+ return NX86::HaveAVX2() && NX86::HaveFMA() && NX86::HaveAVX512BW() && NX86::HaveAVX512VNNI();
+ }
+
Y_UNIT_TEST(TestDotProduct8) {
TVector<i8> a(100);
FillWithRandomNumbers(a.data(), 179, 100);
TVector<i8> b(100);
FillWithRandomNumbers(b.data(), 239, 100);
+ const bool haveVnni = HaveVnniDotProduct();
for (size_t i = 0; i < 30; ++i) {
for (size_t length = 1; length + i + 1 < a.size(); ++length) {
- UNIT_ASSERT_EQUAL(DotProduct(a.data() + i, b.data() + i, length), (SimpleDotProduct<i32, i8>(a.data() + i, b.data() + i, length)));
- UNIT_ASSERT_EQUAL(DotProductSimple(a.data() + i, b.data() + i, length), (SimpleDotProduct<i32, i8>(a.data() + i, b.data() + i, length)));
+ const i32 expected = SimpleDotProduct<i32, i8>(a.data() + i, b.data() + i, length);
+ UNIT_ASSERT_EQUAL(DotProduct(a.data() + i, b.data() + i, length), expected);
+ UNIT_ASSERT_EQUAL(DotProductSimple(a.data() + i, b.data() + i, length), expected);
+ if (haveVnni) {
+ UNIT_ASSERT_EQUAL(DotProductVnni(a.data() + i, b.data() + i, length), expected);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(TestDotProduct8VnniEdges) {
+ if (!HaveVnniDotProduct()) {
+ return;
+ }
+
+ TVector<i8> a(512);
+ TVector<i8> b(512);
+ for (size_t i = 0; i < a.size(); ++i) {
+ a[i] = static_cast<i8>((i * 37) % 256 - 128);
+ b[i] = static_cast<i8>((i * 53 + 17) % 256 - 128);
+ }
+
+ for (size_t offset = 0; offset < 16; ++offset) {
+ for (size_t length = 0; length + offset <= a.size(); ++length) {
+ UNIT_ASSERT_EQUAL(
+ DotProductVnni(a.data() + offset, b.data() + offset, length),
+ (SimpleDotProduct<i32, i8>(a.data() + offset, b.data() + offset, length)));
}
}
}
diff --git a/library/cpp/dot_product/dot_product_vnni.cpp b/library/cpp/dot_product/dot_product_vnni.cpp
new file mode 100644
index 00000000000..4b3c52def49
--- /dev/null
+++ b/library/cpp/dot_product/dot_product_vnni.cpp
@@ -0,0 +1,125 @@
+#include "dot_product_vnni.h"
+#include "dot_product_avx2.h"
+
+#include <util/system/compiler.h>
+#include <util/system/platform.h>
+
+#if defined(__AVX512VNNI__) && defined(__AVX512BW__) && defined(__AVX512F__)
+
+#include <immintrin.h>
+
+namespace {
+ Y_FORCE_INLINE __m512i Load512i(const void* ptr) {
+ return _mm512_loadu_si512(ptr);
+ }
+
+ Y_FORCE_INLINE __m512i CorrectSignedLhsDotProduct(
+ const __m512i biasedDotProduct,
+ const __m512i rhsSum) noexcept
+ {
+ return _mm512_sub_epi32(biasedDotProduct, _mm512_slli_epi32(rhsSum, 7));
+ }
+
+ Y_FORCE_INLINE void AccumulateDotProductI8Vnni(
+ __m512i& dotProduct,
+ const i8* lhs,
+ const i8* rhs,
+ const __m512i zero,
+ const __m512i signBit,
+ const __m512i ones) noexcept
+ {
+ const __m512i l = Load512i(lhs);
+ const __m512i r = Load512i(rhs);
+ const __m512i unsignedL = _mm512_xor_si512(l, signBit);
+
+ // AVX512 VNNI has u8*i8 dot product. For signed lhs we compute
+ // (lhs + 128) * rhs and immediately subtract 128 * sum(rhs).
+ // Correcting each 64-byte chunk keeps the accumulator in the same
+ // range as a regular signed i8 dot product accumulator instead of
+ // accumulating a much larger biased intermediate value.
+ const __m512i biasedDotProduct = _mm512_dpbusd_epi32(zero, unsignedL, r);
+ const __m512i rhsSum = _mm512_dpbusd_epi32(zero, ones, r);
+ dotProduct = _mm512_add_epi32(dotProduct, CorrectSignedLhsDotProduct(biasedDotProduct, rhsSum));
+ }
+
+ Y_FORCE_INLINE void AccumulateTailDotProductI8Vnni(
+ __m512i& dotProduct,
+ const i8* lhs,
+ const i8* rhs,
+ const size_t length,
+ const __m512i zero,
+ const __m512i signBit,
+ const __m512i ones) noexcept
+ {
+ const __mmask64 mask = (ui64(1) << length) - 1;
+ const __m512i l = _mm512_maskz_loadu_epi8(mask, lhs);
+ const __m512i r = _mm512_maskz_loadu_epi8(mask, rhs);
+ const __m512i unsignedL = _mm512_xor_si512(l, signBit);
+
+ const __m512i biasedDotProduct = _mm512_dpbusd_epi32(zero, unsignedL, r);
+ const __m512i rhsSum = _mm512_dpbusd_epi32(zero, ones, r);
+ dotProduct = _mm512_add_epi32(dotProduct, CorrectSignedLhsDotProduct(biasedDotProduct, rhsSum));
+ }
+
+ i32 HsumI32(const __m512i v) noexcept {
+ __m128i sum = _mm512_castsi512_si128(v);
+ sum = _mm_add_epi32(sum, _mm512_extracti32x4_epi32(v, 1));
+ sum = _mm_add_epi32(sum, _mm512_extracti32x4_epi32(v, 2));
+ sum = _mm_add_epi32(sum, _mm512_extracti32x4_epi32(v, 3));
+ const __m128i hi64 = _mm_unpackhi_epi64(sum, sum);
+ const __m128i sum64 = _mm_add_epi32(hi64, sum);
+ const __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ const __m128i sum32 = _mm_add_epi32(sum64, hi32);
+ return _mm_cvtsi128_si32(sum32);
+ }
+}
+
+i32 DotProductVnni(const i8* lhs, const i8* rhs, size_t length) noexcept {
+ if (length < 64) {
+ return DotProductAvx2(lhs, rhs, length);
+ }
+
+ const __m512i zero = _mm512_setzero_si512();
+ const __m512i signBit = _mm512_set1_epi8(-128);
+ const __m512i ones = _mm512_set1_epi8(1);
+
+ __m512i dotProduct0 = zero;
+ __m512i dotProduct1 = zero;
+ __m512i dotProduct2 = zero;
+ __m512i dotProduct3 = zero;
+
+ while (length >= 256) {
+ AccumulateDotProductI8Vnni(dotProduct0, lhs, rhs, zero, signBit, ones);
+ AccumulateDotProductI8Vnni(dotProduct1, lhs + 64, rhs + 64, zero, signBit, ones);
+ AccumulateDotProductI8Vnni(dotProduct2, lhs + 128, rhs + 128, zero, signBit, ones);
+ AccumulateDotProductI8Vnni(dotProduct3, lhs + 192, rhs + 192, zero, signBit, ones);
+ lhs += 256;
+ rhs += 256;
+ length -= 256;
+ }
+
+ while (length >= 64) {
+ AccumulateDotProductI8Vnni(dotProduct0, lhs, rhs, zero, signBit, ones);
+ lhs += 64;
+ rhs += 64;
+ length -= 64;
+ }
+
+ if (length > 0) {
+ AccumulateTailDotProductI8Vnni(dotProduct0, lhs, rhs, length, zero, signBit, ones);
+ }
+
+ const __m512i dotProduct = _mm512_add_epi32(
+ _mm512_add_epi32(dotProduct0, dotProduct1),
+ _mm512_add_epi32(dotProduct2, dotProduct3));
+
+ return HsumI32(dotProduct);
+}
+
+#else
+
+i32 DotProductVnni(const i8* lhs, const i8* rhs, size_t length) noexcept {
+ return DotProductAvx2(lhs, rhs, length);
+}
+
+#endif
diff --git a/library/cpp/dot_product/dot_product_vnni.h b/library/cpp/dot_product/dot_product_vnni.h
new file mode 100644
index 00000000000..7949c28cd37
--- /dev/null
+++ b/library/cpp/dot_product/dot_product_vnni.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <util/system/types.h>
+#include <util/system/compiler.h>
+
+#include <stddef.h>
+
+Y_PURE_FUNCTION
+i32 DotProductVnni(const i8* lhs, const i8* rhs, size_t length) noexcept;
diff --git a/library/cpp/dot_product/ya.make b/library/cpp/dot_product/ya.make
index fea9ba8152c..64d78d7bbdc 100644
--- a/library/cpp/dot_product/ya.make
+++ b/library/cpp/dot_product/ya.make
@@ -8,8 +8,10 @@ SRCS(
IF (USE_SSE4 == "yes" AND OS_LINUX == "yes")
SRC_C_AVX2(dot_product_avx2.cpp -mfma)
+ SRC_C_AVX512(dot_product_vnni.cpp -mavx512vnni)
ELSE()
SRC(dot_product_avx2.cpp)
+ SRC(dot_product_vnni.cpp)
ENDIF()
PEERDIR(
generated by cgit v1.3 (git 2.53.0) at 2026-07-22 09:30:34 +0000