diff options
| author | YDBot <[email protected]> | 2026-07-02 18:29:58 +0000 |
|---|---|---|
| committer | YDBot <[email protected]> | 2026-07-02 18:29:58 +0000 |
| commit | a52ffb856a584a5099e37b9c060da3ba4a3db5fb (patch) | |
| tree | 59c7f09bdc9e52d11ce6ddf8b6fabdfe1e73a8d6 /library/cpp | |
| parent | 6faa15e1fbe202f0fddb7c1ffe9886dc81a0425f (diff) | |
| parent | d5cfc51dd97284c59104c50bb156d1a8d9a09dac (diff) | |
Merge pull request #45225 from ydb-platform/merge-rightlib-260701-1735
Diffstat (limited to 'library/cpp')
29 files changed, 1042 insertions, 66 deletions
diff --git a/library/cpp/charset/ut/ya.make b/library/cpp/charset/ut/ya.make index c22436f8acf..62f3426f67c 100644 --- a/library/cpp/charset/ut/ya.make +++ b/library/cpp/charset/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(library/cpp/charset) +NO_COW() + SRCS( iconv_ut.cpp recyr_int_ut.cpp diff --git a/library/cpp/charset/wide_ut.cpp b/library/cpp/charset/wide_ut.cpp index 93567161bab..4f5a666cb8e 100644 --- a/library/cpp/charset/wide_ut.cpp +++ b/library/cpp/charset/wide_ut.cpp @@ -10,6 +10,8 @@ #include <algorithm> +extern const int TStringUseCow; + namespace { //! three UTF8 encoded russian letters (A, B, V) const char yandexCyrillicAlphabet[] = @@ -264,10 +266,12 @@ void TConversionTest::TestRecodeIntoString() { TUtf16String copy = sUnicode; // increase ref-counter wres = NDetail::Recode<char>(UTF8Text, sUnicode, CODES_UTF8); UNIT_ASSERT(sUnicode == UnicodeText); // same content + if (TStringUseCow) { #ifndef TSTRING_IS_STD_STRING - UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) - UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer + UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) + UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer #endif + } UNIT_ASSERT(sUnicode.size() == wres.size()); // same content } diff --git a/library/cpp/cron_expression/cron_expression.cpp b/library/cpp/cron_expression/cron_expression.cpp index 55b64ee6d24..31a538a0766 100644 --- a/library/cpp/cron_expression/cron_expression.cpp +++ b/library/cpp/cron_expression/cron_expression.cpp @@ -1060,12 +1060,15 @@ private: } value = GetField(calendar, ECronField::CF_DAY_OF_MONTH); + // FindDay may cross a month/year boundary keeping the same day number (e.g. 'L-1'), check them too + const auto prevMonth = calendar.month(); + const auto prevYear = calendar.year(); updateValue = FindDay(calendar, Target_.GetDayInMonth(), value, GetWeekday(calendar), Target_.GetFlags(), resets, offset); if (updateValue.Empty()) { ythrow yexception() << ErrorDateNotExists; } - if (value == updateValue) { + if (value == updateValue && prevMonth == calendar.month() && prevYear == calendar.year()) { resets.Set(static_cast<uint32_t>(ECronField::CF_DAY_OF_MONTH)); } else { continue; diff --git a/library/cpp/cron_expression/ut/cron_expression_ut.cpp b/library/cpp/cron_expression/ut/cron_expression_ut.cpp index 999f1e00260..0a423dd85bd 100644 --- a/library/cpp/cron_expression/ut/cron_expression_ut.cpp +++ b/library/cpp/cron_expression/ut/cron_expression_ut.cpp @@ -349,6 +349,69 @@ Y_UNIT_TEST_SUITE(TestNext) { TestNext("@minutely", "2024-04-22T16:05:32Z", "2024-04-22T16:06:00Z"); TestNext("@secondly", "2021-12-07T12:00:00Z", "2021-12-07T12:00:01Z"); } + + Y_UNIT_TEST(RangesListsAndSteps) { + TestNext("0 0 9-17 * * *", "2021-03-15T08:00:00Z", "2021-03-15T09:00:00Z"); + TestNext("0 0 9-17 * * *", "2021-03-15T20:00:00Z", "2021-03-16T09:00:00Z"); + TestNext("0 0,30 * * * *", "2021-03-15T10:05:00Z", "2021-03-15T10:30:00Z"); + TestNext("0 */15 * * * *", "2021-03-15T10:07:00Z", "2021-03-15T10:15:00Z"); + TestNext("0 0 */6 * * *", "2021-03-15T05:00:00Z", "2021-03-15T06:00:00Z"); + TestNext("0 0 0 1,15 * *", "2021-03-16T00:00:00Z", "2021-04-01T00:00:00Z"); + TestNext("0 0 0 1-7 * *", "2021-03-10T00:00:00Z", "2021-04-01T00:00:00Z"); + TestNext("0 0 0 */10 * *", "2021-03-05T00:00:00Z", "2021-03-11T00:00:00Z"); + TestNext("0 30 9 * * *", "2021-03-15T10:20:30Z", "2021-03-16T09:30:00Z"); + TestNext("0 0 * * * *", "2021-03-15T10:20:30Z", "2021-03-15T11:00:00Z"); + TestNext("15 10 * * *", "2021-03-15T10:20:30Z", "2021-03-16T10:15:00Z"); + } + + Y_UNIT_TEST(MonthsAndNames) { + TestNext("0 0 0 1 1 *", "2021-06-15T00:00:00Z", "2022-01-01T00:00:00Z"); + TestNext("0 0 0 1 JAN *", "2021-06-15T00:00:00Z", "2022-01-01T00:00:00Z"); + TestNext("0 0 0 1 2-4 *", "2021-06-15T00:00:00Z", "2022-02-01T00:00:00Z"); + TestNext("0 0 0 1 DEC *", "2021-12-31T00:00:00Z", "2022-12-01T00:00:00Z"); + TestNext("0 0 0 29 2 *", "2021-03-01T00:00:00Z", "2024-02-29T00:00:00Z"); + } + + Y_UNIT_TEST(DaysOfWeek) { + TestNext("0 0 0 * * 1", "2021-03-15T12:00:00Z", "2021-03-22T00:00:00Z"); + TestNext("0 0 0 * * MON", "2021-03-15T12:00:00Z", "2021-03-22T00:00:00Z"); + TestNext("0 0 0 * * 0", "2021-03-15T12:00:00Z", "2021-03-21T00:00:00Z"); + TestNext("0 0 0 * * 7", "2021-03-15T12:00:00Z", "2021-03-21T00:00:00Z"); + TestNext("0 0 0 * * 6-7", "2021-03-15T12:00:00Z", "2021-03-20T00:00:00Z"); + TestNext("0 0 0 * * MON-FRI", "2021-03-19T12:00:00Z", "2021-03-22T00:00:00Z"); + TestNext("0 0 0 * * SAT,SUN", "2021-03-15T12:00:00Z", "2021-03-20T00:00:00Z"); + TestNext("0 0 12 * * 5", "2021-03-15T12:00:00Z", "2021-03-19T12:00:00Z"); + } + + Y_UNIT_TEST(NthWeekdayOfMonth) { + TestNext("0 0 0 * * 1#1", "2021-03-01T00:00:00Z", "2021-04-05T00:00:00Z"); + TestNext("0 0 0 * * 1#2", "2021-03-01T00:00:00Z", "2021-03-08T00:00:00Z"); + TestNext("0 0 0 * * 5#3", "2021-03-01T00:00:00Z", "2021-03-19T00:00:00Z"); + TestNext("0 0 0 * * 1#-1", "2021-03-01T00:00:00Z", "2021-03-29T00:00:00Z"); + TestNext("0 0 0 * * 3#-2", "2021-03-01T00:00:00Z", "2021-03-24T00:00:00Z"); + } + + Y_UNIT_TEST(LastDaysOfMonthWithTime) { + // 'L-N' rolling into the next month with an unchanged day number must keep the requested time + TestNext("59 59 23 L-1 * *", "2026-06-30T03:14:00Z", "2026-07-30T23:59:59Z"); + TestNext("59 59 23 L-1 * *", "2026-09-30T10:00:00Z", "2026-10-30T23:59:59Z"); + TestNext("59 59 23 L-1 * *", "2026-11-30T10:00:00Z", "2026-12-30T23:59:59Z"); + TestNext("59 59 23 L-1 * *", "2026-04-30T10:00:00Z", "2026-05-30T23:59:59Z"); + TestNext("59 59 23 L-1 * *", "2026-07-01T00:00:00Z", "2026-07-30T23:59:59Z"); + TestNext("0 30 12 L-2 * *", "2026-06-29T13:00:00Z", "2026-07-29T12:30:00Z"); + TestNext("59 59 23 L * *", "2021-03-31T10:00:00Z", "2021-03-31T23:59:59Z"); + TestNext("59 59 23 L * *", "2021-04-30T23:59:59Z", "2021-05-31T23:59:59Z"); + TestNext("59 59 23 L * *", "2025-08-31T05:00:00Z", "2025-08-31T23:59:59Z"); + TestNext("0 0 0 L * *", "2021-02-10T00:00:00Z", "2021-02-28T00:00:00Z"); + TestNext("0 0 0 L * *", "2020-02-10T00:00:00Z", "2020-02-29T00:00:00Z"); + TestNext("0 0 0 L-5 * *", "2021-02-10T00:00:00Z", "2021-02-23T00:00:00Z"); + TestNext("30 0 0 L-1 * *", "2025-08-30T00:00:00Z", "2025-08-30T00:00:30Z"); + } + + Y_UNIT_TEST(YearField) { + TestNext("0 0 0 1 1 * 2030", "2021-01-01T00:00:00Z", "2030-01-01T00:00:00Z"); + TestNext("0 0 0 29 2 * 2024", "2021-01-01T00:00:00Z", "2024-02-29T00:00:00Z"); + } } Y_UNIT_TEST_SUITE(TestPrev) { @@ -441,6 +504,29 @@ Y_UNIT_TEST_SUITE(TestPrev) { TestPrev("30 50 23 20,21,22 * *", "2023-07-22T12:34:56Z", "2023-07-21T23:50:30Z"); TestPrev("30 50 23 20,21,22 * *", "2023-07-23T12:34:56Z", "2023-07-22T23:50:30Z"); } + + Y_UNIT_TEST(MoreCases) { + TestPrev("0 0 0 * * *", "2021-03-15T10:20:30Z", "2021-03-15T00:00:00Z"); + TestPrev("0 0 9-17 * * *", "2021-03-15T08:00:00Z", "2021-03-14T17:00:00Z"); + TestPrev("0 0,30 * * * *", "2021-03-15T10:05:00Z", "2021-03-15T10:00:00Z"); + TestPrev("0 0 0 1 1 *", "2021-06-15T00:00:00Z", "2021-01-01T00:00:00Z"); + TestPrev("0 0 0 29 2 *", "2021-03-01T00:00:00Z", "2020-02-29T00:00:00Z"); + TestPrev("0 0 0 * * MON", "2021-03-15T12:00:00Z", "2021-03-15T00:00:00Z"); + TestPrev("0 0 0 * * SAT,SUN", "2021-03-15T12:00:00Z", "2021-03-14T00:00:00Z"); + TestPrev("0 0 0 * * 1#-1", "2021-03-01T00:00:00Z", "2021-02-22T00:00:00Z"); + TestPrev("0 0 0 L * *", "2021-02-10T00:00:00Z", "2021-01-31T00:00:00Z"); + TestPrev("0 0 0 LW * *", "2021-05-10T00:00:00Z", "2021-04-30T00:00:00Z"); + TestPrev("0 0 0 15W * *", "2021-05-01T00:00:00Z", "2021-04-15T00:00:00Z"); + } + + Y_UNIT_TEST(LastDaysOfMonthWithTime) { + // mirror of the TestNext case in the prev direction + TestPrev("59 59 23 L * *", "2025-08-31T05:00:00Z", "2025-07-31T23:59:59Z"); + TestPrev("30 0 0 L-1 * *", "2025-08-30T00:00:00Z", "2025-07-30T00:00:30Z"); + TestPrev("30 0 0 L-1 * *", "2025-10-30T00:00:00Z", "2025-09-29T00:00:30Z"); + TestPrev("59 59 23 L-1 * *", "2026-06-30T03:14:00Z", "2026-06-29T23:59:59Z"); + TestPrev("0 30 12 L-2 * *", "2026-06-29T13:00:00Z", "2026-06-28T12:30:00Z"); + } } Y_UNIT_TEST_SUITE(AliceBasicUsageExamples) { diff --git a/library/cpp/dot_product/common.h b/library/cpp/dot_product/common.h index 25cd6153404..bd72a6ed3df 100644 --- a/library/cpp/dot_product/common.h +++ b/library/cpp/dot_product/common.h @@ -18,3 +18,9 @@ enum class ETriWayDotProductComputeMask: unsigned { Left = 0b110, // skip computation of R·R Right = 0b011, // skip computation of L·L }; + +struct TTriWayDotProductFloatI8 { + float LL = 0; + float LR = 0; + float RR = 0; +}; diff --git a/library/cpp/dot_product/dot_product.cpp b/library/cpp/dot_product/dot_product.cpp index 79ccbdb3b7d..003dbe88335 100644 --- a/library/cpp/dot_product/dot_product.cpp +++ b/library/cpp/dot_product/dot_product.cpp @@ -15,11 +15,18 @@ namespace NDotProductImpl { ui32 (*DotProductUi8Impl)(const ui8* lhs, const ui8* rhs, size_t length) noexcept = &DotProductSimple; i64 (*DotProductI32Impl)(const i32* lhs, const i32* rhs, size_t length) noexcept = &DotProductSimple; float (*DotProductFloatImpl)(const float* lhs, const float* rhs, size_t length) noexcept = &DotProductSimple; + float (*DotProductFloatI8Impl)(const float* lhs, const i8* rhs, size_t length) noexcept = &DotProductSimple; double (*DotProductDoubleImpl)(const double* lhs, const double* rhs, size_t length) noexcept = &DotProductSimple; TTriWayDotProduct<float> (*TriWayDotProductImpl) (const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept = &TriWayDotProductSimple; + TTriWayDotProductFloatI8 (*TriWayDotProductFloatI8Impl) + (const float* lhs, const i8* rhs, size_t length) noexcept = &TriWayDotProductFloatI8Simple; + + TTriWayDotProduct<i32> (*TriWayDotProductI8Impl) + (const i8* lhs, const i8* rhs, size_t length) noexcept = &TriWayDotProductI8Simple; + namespace { [[maybe_unused]] const int _ = [] { @@ -28,16 +35,22 @@ namespace NDotProductImpl { DotProductUi8Impl = &DotProductAvx2; DotProductI32Impl = &DotProductAvx2; DotProductFloatImpl = &DotProductAvx2; + DotProductFloatI8Impl = &DotProductFloatI8Avx2; DotProductDoubleImpl = &DotProductAvx2; TriWayDotProductImpl = &TriWayDotProductAvx2; + TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Avx2; + TriWayDotProductI8Impl = &TriWayDotProductI8Avx2; } else { #ifdef ARCADIA_SSE DotProductI8Impl = &DotProductSse; DotProductUi8Impl = &DotProductSse; DotProductI32Impl = &DotProductSse; DotProductFloatImpl = &DotProductSse; + DotProductFloatI8Impl = &DotProductSse; DotProductDoubleImpl = &DotProductSse; TriWayDotProductImpl = &TriWayDotProductSse; + TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Sse; + TriWayDotProductI8Impl = &TriWayDotProductI8Sse; #endif } return 0; @@ -171,15 +184,21 @@ namespace NDotProduct { NDotProductImpl::DotProductUi8Impl = &DotProductSse; NDotProductImpl::DotProductI32Impl = &DotProductSse; NDotProductImpl::DotProductFloatImpl = &DotProductSse; + NDotProductImpl::DotProductFloatI8Impl = &DotProductSse; NDotProductImpl::DotProductDoubleImpl = &DotProductSse; NDotProductImpl::TriWayDotProductImpl = &TriWayDotProductSse; + NDotProductImpl::TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Sse; + NDotProductImpl::TriWayDotProductI8Impl = &TriWayDotProductI8Sse; #else NDotProductImpl::DotProductI8Impl = &DotProductSimple; NDotProductImpl::DotProductUi8Impl = &DotProductSimple; NDotProductImpl::DotProductI32Impl = &DotProductSimple; NDotProductImpl::DotProductFloatImpl = &DotProductSimple; + NDotProductImpl::DotProductFloatI8Impl = &DotProductSimple; NDotProductImpl::DotProductDoubleImpl = &DotProductSimple; NDotProductImpl::TriWayDotProductImpl = &TriWayDotProductSimple; + NDotProductImpl::TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Simple; + NDotProductImpl::TriWayDotProductI8Impl = &TriWayDotProductI8Simple; #endif } } diff --git a/library/cpp/dot_product/dot_product.h b/library/cpp/dot_product/dot_product.h index 154ffaf38f6..cc13ae5d26b 100644 --- a/library/cpp/dot_product/dot_product.h +++ b/library/cpp/dot_product/dot_product.h @@ -15,10 +15,17 @@ namespace NDotProductImpl { extern ui32 (*DotProductUi8Impl)(const ui8* lhs, const ui8* rhs, size_t length) noexcept; extern i64 (*DotProductI32Impl)(const i32* lhs, const i32* rhs, size_t length) noexcept; extern float (*DotProductFloatImpl)(const float* lhs, const float* rhs, size_t length) noexcept; + extern float (*DotProductFloatI8Impl)(const float* lhs, const i8* rhs, size_t length) noexcept; extern double (*DotProductDoubleImpl)(const double* lhs, const double* rhs, size_t length) noexcept; extern TTriWayDotProduct<float> (*TriWayDotProductImpl) (const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept; + + extern TTriWayDotProductFloatI8 (*TriWayDotProductFloatI8Impl) + (const float* lhs, const i8* rhs, size_t length) noexcept; + + extern TTriWayDotProduct<i32> (*TriWayDotProductI8Impl) + (const i8* lhs, const i8* rhs, size_t length) noexcept; } Y_PURE_FUNCTION @@ -42,6 +49,11 @@ inline float DotProduct(const float* lhs, const float* rhs, size_t length) noexc } Y_PURE_FUNCTION +inline float DotProduct(const float* lhs, const i8* rhs, size_t length) noexcept { + return NDotProductImpl::DotProductFloatI8Impl(lhs, rhs, length); +} + +Y_PURE_FUNCTION inline double DotProduct(const double* lhs, const double* rhs, size_t length) noexcept { return NDotProductImpl::DotProductDoubleImpl(lhs, rhs, length); } @@ -71,6 +83,24 @@ static inline TTriWayDotProduct<float> TriWayDotProduct( return TriWayDotProduct(lhs, rhs, length, static_cast<unsigned>(mask)); } +Y_PURE_FUNCTION +inline TTriWayDotProductFloatI8 TriWayDotProduct( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + return NDotProductImpl::TriWayDotProductFloatI8Impl(lhs, rhs, length); +} + +Y_PURE_FUNCTION +inline TTriWayDotProduct<i32> TriWayDotProduct( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + return NDotProductImpl::TriWayDotProductI8Impl(lhs, rhs, length); +} + namespace NDotProduct { // Simpler wrapper allowing to use this functions as template argument. template <typename T> diff --git a/library/cpp/dot_product/dot_product_avx2.cpp b/library/cpp/dot_product/dot_product_avx2.cpp index c95888ecda0..0d2c103feea 100644 --- a/library/cpp/dot_product/dot_product_avx2.cpp +++ b/library/cpp/dot_product/dot_product_avx2.cpp @@ -155,6 +155,71 @@ namespace { return HsumI32(sum); } + + template <class TInput, class TExtend> + TTriWayDotProduct<i32> TriWayDotProductInt8Avx2( + const TInput* lhs, + const TInput* rhs, + size_t length, + TExtend extend) noexcept + { + __m256i sumLL = _mm256_setzero_si256(); + __m256i sumLR = _mm256_setzero_si256(); + __m256i sumRR = _mm256_setzero_si256(); + + if (const auto leftover = length % 16; leftover != 0) { + const auto a = _mm_blendv_epi8( + Load128i(lhs), _mm_setzero_ps(), BlendMask8[leftover]); + const auto b = _mm_blendv_epi8( + Load128i(rhs), _mm_setzero_ps(), BlendMask8[leftover]); + const auto l = extend(a); + const auto r = extend(b); + sumLL = _mm256_add_epi32(sumLL, _mm256_madd_epi16(l, l)); + sumLR = _mm256_add_epi32(sumLR, _mm256_madd_epi16(l, r)); + sumRR = _mm256_add_epi32(sumRR, _mm256_madd_epi16(r, r)); + lhs += leftover; + rhs += leftover; + length -= leftover; + } + + while (length >= 32) { + const auto l0 = extend(Load128i(lhs)); + const auto r0 = extend(Load128i(rhs)); + const auto l1 = extend(Load128i(lhs + 16)); + const auto r1 = extend(Load128i(rhs + 16)); + sumLL = _mm256_add_epi32(sumLL, _mm256_add_epi32(_mm256_madd_epi16(l0, l0), _mm256_madd_epi16(l1, l1))); + sumLR = _mm256_add_epi32(sumLR, _mm256_add_epi32(_mm256_madd_epi16(l0, r0), _mm256_madd_epi16(l1, r1))); + sumRR = _mm256_add_epi32(sumRR, _mm256_add_epi32(_mm256_madd_epi16(r0, r0), _mm256_madd_epi16(r1, r1))); + lhs += 32; + rhs += 32; + length -= 32; + } + + if (length >= 16) { + const auto l = extend(Load128i(lhs)); + const auto r = extend(Load128i(rhs)); + sumLL = _mm256_add_epi32(sumLL, _mm256_madd_epi16(l, l)); + sumLR = _mm256_add_epi32(sumLR, _mm256_madd_epi16(l, r)); + sumRR = _mm256_add_epi32(sumRR, _mm256_madd_epi16(r, r)); + lhs += 16; + rhs += 16; + length -= 16; + } + + TTriWayDotProduct<i32> result{0, 0, 0}; + result.LL = HsumI32(sumLL); + result.LR = HsumI32(sumLR); + result.RR = HsumI32(sumRR); + + for (size_t i = 0; i < length; ++i) { + const i32 l = lhs[i]; + const i32 r = rhs[i]; + result.LL += l * l; + result.LR += l * r; + result.RR += r * r; + } + return result; + } } i32 DotProductAvx2(const i8* lhs, const i8* rhs, size_t length) noexcept { @@ -403,6 +468,127 @@ TTriWayDotProduct<float> TriWayDotProductAvx2( } } +Y_FORCE_INLINE __m256 LoadFloatI8Rhs8Avx2(const i8* rhs) { + const __m128i rhsBytes = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rhs)); + const __m128i rhsI16 = _mm_cvtepi8_epi16(rhsBytes); + const __m128 rhsLo = _mm256_castps256_ps128(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(rhsI16))); + const __m128 rhsHi = _mm256_castps256_ps128(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm_srli_si128(rhsI16, 8)))); + return _mm256_set_m128(rhsHi, rhsLo); +} + +Y_FORCE_INLINE void TriWayDotProductFloatI8IterationAvx2( + __m256& sumLL, + __m256& sumLR, + __m256& sumRR, + const float* lhs, + const i8* rhs) noexcept +{ + const __m256 l = _mm256_loadu_ps(lhs); + const __m256 r = LoadFloatI8Rhs8Avx2(rhs); + sumLL = _mm256_fmadd_ps(l, l, sumLL); + sumLR = _mm256_fmadd_ps(l, r, sumLR); + sumRR = _mm256_fmadd_ps(r, r, sumRR); +} + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + __m256 sumLL1 = _mm256_setzero_ps(); + __m256 sumLR1 = _mm256_setzero_ps(); + __m256 sumRR1 = _mm256_setzero_ps(); + __m256 sumLL2 = _mm256_setzero_ps(); + __m256 sumLR2 = _mm256_setzero_ps(); + __m256 sumRR2 = _mm256_setzero_ps(); + + while (length >= 16) { + TriWayDotProductFloatI8IterationAvx2(sumLL1, sumLR1, sumRR1, lhs, rhs); + TriWayDotProductFloatI8IterationAvx2(sumLL2, sumLR2, sumRR2, lhs + 8, rhs + 8); + lhs += 16; + rhs += 16; + length -= 16; + } + + if (length >= 8) { + TriWayDotProductFloatI8IterationAvx2(sumLL1, sumLR1, sumRR1, lhs, rhs); + lhs += 8; + rhs += 8; + length -= 8; + } + + sumLL1 = _mm256_add_ps(sumLL1, sumLL2); + sumLR1 = _mm256_add_ps(sumLR1, sumLR2); + sumRR1 = _mm256_add_ps(sumRR1, sumRR2); + + const auto hsumPs = [](__m256 v) { + v = _mm256_add_ps(v, _mm256_permute2f128_ps(v, v, 1)); + v = _mm256_hadd_ps(v, v); + return _mm256_cvtss_f32(_mm256_hadd_ps(v, v)); + }; + + TTriWayDotProductFloatI8 result; + result.LL = hsumPs(sumLL1); + result.LR = hsumPs(sumLR1); + result.RR = hsumPs(sumRR1); + + for (size_t i = 0; i < length; ++i) { + const float l = lhs[i]; + const float r = rhs[i]; + result.LL += l * l; + result.LR += l * r; + result.RR += r * r; + } + return result; +} + +float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept { + if (length < 8) { + return DotProductSse(lhs, rhs, length); + } + + __m256 sum1 = _mm256_setzero_ps(); + __m256 sum2 = _mm256_setzero_ps(); + + while (length >= 16) { + sum1 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), LoadFloatI8Rhs8Avx2(rhs), sum1); + sum2 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs + 8), LoadFloatI8Rhs8Avx2(rhs + 8), sum2); + lhs += 16; + rhs += 16; + length -= 16; + } + + if (length >= 8) { + sum1 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), LoadFloatI8Rhs8Avx2(rhs), sum1); + lhs += 8; + rhs += 8; + length -= 8; + } + + __m256 sum = _mm256_add_ps(sum1, sum2); + sum = _mm256_add_ps(sum, _mm256_permute2f128_ps(sum, sum, 1)); + sum = _mm256_hadd_ps(sum, sum); + float result = _mm256_cvtss_f32(_mm256_hadd_ps(sum, sum)); + + for (size_t i = 0; i < length; ++i) { + result += lhs[i] * rhs[i]; + } + return result; +} + +TTriWayDotProduct<i32> TriWayDotProductI8Avx2( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + if (length < 16) { + return TriWayDotProductI8Sse(lhs, rhs, length); + } + return TriWayDotProductInt8Avx2(lhs, rhs, length, [](const __m128i x) { + return _mm256_cvtepi8_epi16(x); + }); +} + #elif defined(ARCADIA_SSE) i32 DotProductAvx2(const i8* lhs, const i8* rhs, size_t length) noexcept { @@ -434,6 +620,26 @@ TTriWayDotProduct<float> TriWayDotProductAvx2( return TriWayDotProductSse(lhs, rhs, length, computeRR); } +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + return TriWayDotProductFloatI8Sse(lhs, rhs, length); +} + +float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept { + return DotProductSse(lhs, rhs, length); +} + +TTriWayDotProduct<i32> TriWayDotProductI8Avx2( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + return TriWayDotProductI8Sse(lhs, rhs, length); +} + #else i32 DotProductAvx2(const i8* lhs, const i8* rhs, size_t length) noexcept { @@ -465,4 +671,24 @@ TTriWayDotProduct<float> TriWayDotProductAvx2( return TriWayDotProductSimple(lhs, rhs, length, computeRR); } +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + return TriWayDotProductFloatI8Simple(lhs, rhs, length); +} + +float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept { + return DotProductSimple(lhs, rhs, length); +} + +TTriWayDotProduct<i32> TriWayDotProductI8Avx2( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + return TriWayDotProductI8Simple(lhs, rhs, length); +} + #endif diff --git a/library/cpp/dot_product/dot_product_avx2.h b/library/cpp/dot_product/dot_product_avx2.h index 68af7e6c8c2..d99dc5a1709 100644 --- a/library/cpp/dot_product/dot_product_avx2.h +++ b/library/cpp/dot_product/dot_product_avx2.h @@ -18,8 +18,21 @@ Y_PURE_FUNCTION float DotProductAvx2(const float* lhs, const float* rhs, size_t length) noexcept; Y_PURE_FUNCTION +float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept; + +Y_PURE_FUNCTION double DotProductAvx2(const double* lhs, const double* rhs, size_t length) noexcept; Y_PURE_FUNCTION TTriWayDotProduct<float> TriWayDotProductAvx2 (const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept; + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2( + const float* lhs, + const i8* rhs, + size_t length) noexcept; + +TTriWayDotProduct<i32> TriWayDotProductI8Avx2( + const i8* lhs, + const i8* rhs, + size_t length) noexcept; diff --git a/library/cpp/dot_product/dot_product_simple.cpp b/library/cpp/dot_product/dot_product_simple.cpp index 50afcfe62a3..454dcb3ab0e 100644 --- a/library/cpp/dot_product/dot_product_simple.cpp +++ b/library/cpp/dot_product/dot_product_simple.cpp @@ -30,6 +30,14 @@ float DotProductSimple(const float* lhs, const float* rhs, size_t length) noexce return DotProductSimpleImpl<float, float>(lhs, rhs, length); } +float DotProductSimple(const float* lhs, const i8* rhs, size_t length) noexcept { + float sum = 0; + for (size_t i = 0; i < length; ++i) { + sum += lhs[i] * rhs[i]; + } + return sum; +} + double DotProductSimple(const double* lhs, const double* rhs, size_t length) noexcept { return DotProductSimpleImpl<double, double>(lhs, rhs, length); } @@ -74,3 +82,35 @@ TTriWayDotProduct<float> TriWayDotProductSimple( } return result; } + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Simple( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + TTriWayDotProductFloatI8 result; + for (size_t i = 0; i < length; ++i) { + const float l = lhs[i]; + const float r = rhs[i]; + result.LL += l * l; + result.LR += l * r; + result.RR += r * r; + } + return result; +} + +TTriWayDotProduct<i32> TriWayDotProductI8Simple( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + TTriWayDotProduct<i32> result{0, 0, 0}; + for (size_t i = 0; i < length; ++i) { + const i32 l = lhs[i]; + const i32 r = rhs[i]; + result.LL += l * l; + result.LR += l * r; + result.RR += r * r; + } + return result; +} diff --git a/library/cpp/dot_product/dot_product_simple.h b/library/cpp/dot_product/dot_product_simple.h index f951d05f5f0..d0eac9a24b0 100644 --- a/library/cpp/dot_product/dot_product_simple.h +++ b/library/cpp/dot_product/dot_product_simple.h @@ -35,6 +35,9 @@ Y_PURE_FUNCTION float DotProductSimple(const float* lhs, const float* rhs, size_t length) noexcept; Y_PURE_FUNCTION +float DotProductSimple(const float* lhs, const i8* rhs, size_t length) noexcept; + +Y_PURE_FUNCTION double DotProductSimple(const double* lhs, const double* rhs, size_t length) noexcept; Y_PURE_FUNCTION @@ -46,3 +49,13 @@ TTriWayDotProduct<float> TriWayDotProductSimple( const float* rhs, size_t length, bool computeRR) noexcept; + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Simple( + const float* lhs, + const i8* rhs, + size_t length) noexcept; + +TTriWayDotProduct<i32> TriWayDotProductI8Simple( + const i8* lhs, + const i8* rhs, + size_t length) noexcept; diff --git a/library/cpp/dot_product/dot_product_sse.cpp b/library/cpp/dot_product/dot_product_sse.cpp index 86f44734876..1cdea0a23c9 100644 --- a/library/cpp/dot_product/dot_product_sse.cpp +++ b/library/cpp/dot_product/dot_product_sse.cpp @@ -1,4 +1,5 @@ #include "dot_product_sse.h" +#include "dot_product_simple.h" #include <library/cpp/sse/sse.h> #include <util/system/platform.h> @@ -173,6 +174,42 @@ float DotProductSse(const float* lhs, const float* rhs, size_t length) noexcept return res[0] + res[1] + res[2] + res[3]; } +float DotProductSse(const float* lhs, const i8* rhs, size_t length) noexcept { +#ifdef _sse4_1_ + float result = 0; + __m128 sum0 = _mm_setzero_ps(); + __m128 sum1 = _mm_setzero_ps(); + + while (length >= 8) { + const __m128 lhs0 = _mm_loadu_ps(lhs); + const __m128 lhs1 = _mm_loadu_ps(lhs + 4); + + const __m128i rhsBytes = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rhs)); + const __m128i rhsI16 = _mm_cvtepi8_epi16(rhsBytes); + const __m128 rhs0 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(rhsI16)); + const __m128 rhs1 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_srli_si128(rhsI16, 8))); + + sum0 = _mm_add_ps(sum0, _mm_mul_ps(lhs0, rhs0)); + sum1 = _mm_add_ps(sum1, _mm_mul_ps(lhs1, rhs1)); + + lhs += 8; + rhs += 8; + length -= 8; + } + + alignas(16) float partial[4]; + _mm_store_ps(partial, _mm_add_ps(sum0, sum1)); + result = partial[0] + partial[1] + partial[2] + partial[3]; + + for (size_t i = 0; i < length; ++i) { + result += lhs[i] * rhs[i]; + } + return result; +#else + return DotProductSimple(lhs, rhs, length); +#endif +} + double DotProductSse(const double* lhs, const double* rhs, size_t length) noexcept { __m128d sum1 = _mm_setzero_pd(); __m128d sum2 = _mm_setzero_pd(); @@ -315,4 +352,130 @@ TTriWayDotProduct<float> TriWayDotProductSse( } } +#ifdef _sse4_1_ + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Sse( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + __m128 sumLL0 = _mm_setzero_ps(); + __m128 sumLR0 = _mm_setzero_ps(); + __m128 sumRR0 = _mm_setzero_ps(); + __m128 sumLL1 = _mm_setzero_ps(); + __m128 sumLR1 = _mm_setzero_ps(); + __m128 sumRR1 = _mm_setzero_ps(); + + while (length >= 8) { + const __m128 lhs0 = _mm_loadu_ps(lhs); + const __m128 lhs1 = _mm_loadu_ps(lhs + 4); + + const __m128i rhsBytes = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rhs)); + const __m128i rhsI16 = _mm_cvtepi8_epi16(rhsBytes); + const __m128 rhs0 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(rhsI16)); + const __m128 rhs1 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_srli_si128(rhsI16, 8))); + + sumLL0 = _mm_add_ps(sumLL0, _mm_mul_ps(lhs0, lhs0)); + sumLR0 = _mm_add_ps(sumLR0, _mm_mul_ps(lhs0, rhs0)); + sumRR0 = _mm_add_ps(sumRR0, _mm_mul_ps(rhs0, rhs0)); + + sumLL1 = _mm_add_ps(sumLL1, _mm_mul_ps(lhs1, lhs1)); + sumLR1 = _mm_add_ps(sumLR1, _mm_mul_ps(lhs1, rhs1)); + sumRR1 = _mm_add_ps(sumRR1, _mm_mul_ps(rhs1, rhs1)); + + lhs += 8; + rhs += 8; + length -= 8; + } + + sumLL0 = _mm_add_ps(sumLL0, sumLL1); + sumLR0 = _mm_add_ps(sumLR0, sumLR1); + sumRR0 = _mm_add_ps(sumRR0, sumRR1); + + alignas(16) float partial[4]; + _mm_store_ps(partial, sumLL0); + TTriWayDotProductFloatI8 result; + result.LL = partial[0] + partial[1] + partial[2] + partial[3]; + _mm_store_ps(partial, sumLR0); + result.LR = partial[0] + partial[1] + partial[2] + partial[3]; + _mm_store_ps(partial, sumRR0); + result.RR = partial[0] + partial[1] + partial[2] + partial[3]; + + for (size_t i = 0; i < length; ++i) { + const float l = lhs[i]; + const float r = rhs[i]; + result.LL += l * l; + result.LR += l * r; + result.RR += r * r; + } + return result; +} + +TTriWayDotProduct<i32> TriWayDotProductI8Sse( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + const __m128i zero = _mm_setzero_si128(); + __m128i sumLL = zero; + __m128i sumLR = zero; + __m128i sumRR = zero; + + while (length >= 16) { + const __m128i lVec = _mm_loadu_si128(reinterpret_cast<const __m128i*>(lhs)); + const __m128i rVec = _mm_loadu_si128(reinterpret_cast<const __m128i*>(rhs)); + + const __m128i lLo = _mm_cvtepi8_epi16(lVec); + const __m128i rLo = _mm_cvtepi8_epi16(rVec); + const __m128i lHi = _mm_cvtepi8_epi16(_mm_alignr_epi8(lVec, lVec, 8)); + const __m128i rHi = _mm_cvtepi8_epi16(_mm_alignr_epi8(rVec, rVec, 8)); + + sumLL = _mm_add_epi32(sumLL, _mm_add_epi32(_mm_madd_epi16(lLo, lLo), _mm_madd_epi16(lHi, lHi))); + sumLR = _mm_add_epi32(sumLR, _mm_add_epi32(_mm_madd_epi16(lLo, rLo), _mm_madd_epi16(lHi, rHi))); + sumRR = _mm_add_epi32(sumRR, _mm_add_epi32(_mm_madd_epi16(rLo, rLo), _mm_madd_epi16(rHi, rHi))); + + lhs += 16; + rhs += 16; + length -= 16; + } + + alignas(16) i32 partial[4]; + _mm_store_si128(reinterpret_cast<__m128i*>(partial), sumLL); + TTriWayDotProduct<i32> result; + result.LL = partial[0] + partial[1] + partial[2] + partial[3]; + _mm_store_si128(reinterpret_cast<__m128i*>(partial), sumLR); + result.LR = partial[0] + partial[1] + partial[2] + partial[3]; + _mm_store_si128(reinterpret_cast<__m128i*>(partial), sumRR); + result.RR = partial[0] + partial[1] + partial[2] + partial[3]; + + for (size_t i = 0; i < length; ++i) { + const i32 l = lhs[i]; + const i32 r = rhs[i]; + result.LL += l * l; + result.LR += l * r; + result.RR += r * r; + } + return result; +} + +#else + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Sse( + const float* lhs, + const i8* rhs, + size_t length) noexcept +{ + return TriWayDotProductFloatI8Simple(lhs, rhs, length); +} + +TTriWayDotProduct<i32> TriWayDotProductI8Sse( + const i8* lhs, + const i8* rhs, + size_t length) noexcept +{ + return TriWayDotProductI8Simple(lhs, rhs, length); +} + +#endif + #endif // ARCADIA_SSE diff --git a/library/cpp/dot_product/dot_product_sse.h b/library/cpp/dot_product/dot_product_sse.h index 8bc26ef75b0..9dbe0da3fd5 100644 --- a/library/cpp/dot_product/dot_product_sse.h +++ b/library/cpp/dot_product/dot_product_sse.h @@ -18,8 +18,21 @@ Y_PURE_FUNCTION float DotProductSse(const float* lhs, const float* rhs, size_t length) noexcept; Y_PURE_FUNCTION +float DotProductSse(const float* lhs, const i8* rhs, size_t length) noexcept; + +Y_PURE_FUNCTION double DotProductSse(const double* lhs, const double* rhs, size_t length) noexcept; Y_PURE_FUNCTION TTriWayDotProduct<float> TriWayDotProductSse (const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept; + +TTriWayDotProductFloatI8 TriWayDotProductFloatI8Sse( + const float* lhs, + const i8* rhs, + size_t length) noexcept; + +TTriWayDotProduct<i32> TriWayDotProductI8Sse( + const i8* lhs, + const i8* rhs, + size_t length) noexcept; diff --git a/library/cpp/dot_product/dot_product_ut.cpp b/library/cpp/dot_product/dot_product_ut.cpp index caf74a832d0..6d07d1a3a31 100644 --- a/library/cpp/dot_product/dot_product_ut.cpp +++ b/library/cpp/dot_product/dot_product_ut.cpp @@ -1,11 +1,13 @@ #include "dot_product.h" #include "dot_product_simple.h" +#include "dot_product_avx2.h" #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/sse/sse.h> #include <util/generic/vector.h> #include <util/random/fast.h> +#include <util/system/cpu_id.h> #include <cmath> @@ -37,6 +39,15 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) { return sum; } + template <class Res, class TLhs, class TRhs> + Res SimpleDotProductMixed(const TLhs* lhs, const TRhs* rhs, size_t length) { + Res sum = 0; + for (size_t i = 0; i < length; ++i) { + sum += static_cast<Res>(lhs[i]) * static_cast<Res>(rhs[i]); + } + return sum; + } + Y_UNIT_TEST(TestDotProduct8) { TVector<i8> a(100); FillWithRandomNumbers(a.data(), 179, 100); @@ -93,6 +104,72 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) { } } + Y_UNIT_TEST(TestDotProductFloatI8) { + TVector<float> floats(100); + FillWithRandomFloats(floats.data(), 179, 100); + TVector<i8> bytes(100); + FillWithRandomNumbers(bytes.data(), 239, 100); + + constexpr double MIXED_EPSILON = 1e-4; + for (size_t i = 0; i < 30; ++i) { + for (size_t length = 1; length + i + 1 < floats.size(); ++length) { + const float expected = SimpleDotProductMixed<float, float, i8>(floats.data() + i, bytes.data() + i, length); + UNIT_ASSERT(std::fabs(DotProduct(floats.data() + i, bytes.data() + i, length) - expected) < MIXED_EPSILON); + UNIT_ASSERT(std::fabs(DotProductSimple(floats.data() + i, bytes.data() + i, length) - expected) < EPSILON); + if (NX86::HaveAVX2() && NX86::HaveFMA()) { + UNIT_ASSERT(std::fabs(DotProductFloatI8Avx2(floats.data() + i, bytes.data() + i, length) - expected) < MIXED_EPSILON); + } + } + } + } + + Y_UNIT_TEST(TestTriWayDotProductFloatI8) { + TVector<float> floats(100); + FillWithRandomFloats(floats.data(), 179, 100); + TVector<i8> bytes(100); + FillWithRandomNumbers(bytes.data(), 239, 100); + + constexpr double MIXED_EPSILON = 1e-4; + for (size_t i = 0; i < 30; ++i) { + for (size_t length = 1; length + i + 1 < floats.size(); ++length) { + const auto expected = TriWayDotProductFloatI8Simple(floats.data() + i, bytes.data() + i, length); + const auto actual = TriWayDotProduct(floats.data() + i, bytes.data() + i, length); + UNIT_ASSERT_DOUBLES_EQUAL(expected.LL, actual.LL, MIXED_EPSILON); + UNIT_ASSERT_DOUBLES_EQUAL(expected.LR, actual.LR, MIXED_EPSILON); + UNIT_ASSERT_DOUBLES_EQUAL(expected.RR, actual.RR, MIXED_EPSILON); + if (NX86::HaveAVX2() && NX86::HaveFMA()) { + const auto avx2 = TriWayDotProductFloatI8Avx2(floats.data() + i, bytes.data() + i, length); + UNIT_ASSERT_DOUBLES_EQUAL(expected.LL, avx2.LL, MIXED_EPSILON); + UNIT_ASSERT_DOUBLES_EQUAL(expected.LR, avx2.LR, MIXED_EPSILON); + UNIT_ASSERT_DOUBLES_EQUAL(expected.RR, avx2.RR, MIXED_EPSILON); + } + } + } + } + + Y_UNIT_TEST(TestTriWayDotProductI8) { + TVector<i8> a(100); + FillWithRandomNumbers(a.data(), 179, 100); + TVector<i8> b(100); + FillWithRandomNumbers(b.data(), 239, 100); + + for (size_t i = 0; i < 30; ++i) { + for (size_t length = 1; length + i + 1 < a.size(); ++length) { + const auto expected = TriWayDotProductI8Simple(a.data() + i, b.data() + i, length); + const auto actual = TriWayDotProduct(a.data() + i, b.data() + i, length); + UNIT_ASSERT_EQUAL(expected.LL, actual.LL); + UNIT_ASSERT_EQUAL(expected.LR, actual.LR); + UNIT_ASSERT_EQUAL(expected.RR, actual.RR); + if (NX86::HaveAVX2() && NX86::HaveFMA()) { + const auto avx2 = TriWayDotProductI8Avx2(a.data() + i, b.data() + i, length); + UNIT_ASSERT_EQUAL(expected.LL, avx2.LL); + UNIT_ASSERT_EQUAL(expected.LR, avx2.LR); + UNIT_ASSERT_EQUAL(expected.RR, avx2.RR); + } + } + } + } + Y_UNIT_TEST(TestL2NormSqaredf) { TVector<float> a(100); FillWithRandomFloats(a.data(), 179, 100); @@ -172,12 +249,14 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) { UNIT_ASSERT_EQUAL(DotProduct(static_cast<const i8*>(nullptr), nullptr, 0), 0); UNIT_ASSERT_EQUAL(DotProduct(static_cast<const ui8*>(nullptr), nullptr, 0), 0); UNIT_ASSERT_EQUAL(DotProduct(static_cast<const i32*>(nullptr), nullptr, 0), 0); - UNIT_ASSERT(std::abs(DotProduct(static_cast<const float*>(nullptr), nullptr, 0)) < EPSILON); + UNIT_ASSERT(std::abs(DotProduct(static_cast<const float*>(nullptr), static_cast<const float*>(nullptr), 0)) < EPSILON); + UNIT_ASSERT(std::abs(DotProduct(static_cast<const float*>(nullptr), static_cast<const i8*>(nullptr), 0)) < EPSILON); UNIT_ASSERT(std::abs(DotProduct(static_cast<const double*>(nullptr), nullptr, 0)) < EPSILON); UNIT_ASSERT_EQUAL(DotProductSimple(static_cast<const i8*>(nullptr), nullptr, 0), 0); UNIT_ASSERT_EQUAL(DotProductSimple(static_cast<const ui8*>(nullptr), nullptr, 0), 0); UNIT_ASSERT_EQUAL(DotProductSimple(static_cast<const i32*>(nullptr), nullptr, 0), 0); - UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const float*>(nullptr), nullptr, 0)) < EPSILON); + UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const float*>(nullptr), static_cast<const float*>(nullptr), 0)) < EPSILON); + UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const float*>(nullptr), static_cast<const i8*>(nullptr), 0)) < EPSILON); UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const double*>(nullptr), nullptr, 0)) < EPSILON); } diff --git a/library/cpp/http/server/conn.cpp b/library/cpp/http/server/conn.cpp index 38a76c4c309..4b72c3f77f0 100644 --- a/library/cpp/http/server/conn.cpp +++ b/library/cpp/http/server/conn.cpp @@ -1,47 +1,38 @@ #include "conn.h" #include <util/network/socket.h> -#include <util/stream/buffered.h> -class THttpServerConn::TImpl { -public: - inline TImpl(const TSocket& s, size_t outputBufferSize) - : S_(s) - , SI_(S_) - , SO_(S_) - , BO_(&SO_, outputBufferSize) - , HI_(&SI_) - , HO_(&BO_, &HI_) - { - } - - inline ~TImpl() { - } - - inline THttpInput* Input() noexcept { - return &HI_; - } +namespace { + class TBlockingSocketStreams: public THttpServerConn::ISocketStreams { + public: + explicit TBlockingSocketStreams(const TSocket& s) + : Socket_(s) + , Input_(Socket_) + , Output_(Socket_) + { + } - inline THttpOutput* Output() noexcept { - return &HO_; - } + IInputStream* Input() override { + return &Input_; + } - inline void Reset() { - if (S_ != INVALID_SOCKET) { - // send RST packet to client - S_.SetLinger(true, 0); - S_.Close(); + IOutputStream* Output() override { + return &Output_; } - } -private: - TSocket S_; - TSocketInput SI_; - TSocketOutput SO_; - TBufferedOutput BO_; - THttpInput HI_; - THttpOutput HO_; -}; + void Reset() override { + if (Socket_ != INVALID_SOCKET) { + // send RST packet to client + Socket_.SetLinger(true, 0); + Socket_.Close(); + } + } + private: + TSocket Socket_; + TSocketInput Input_; + TSocketOutput Output_; + }; +} THttpServerConn::THttpServerConn(const TSocket& s) : THttpServerConn(s, s.MaximumTransferUnit()) @@ -49,21 +40,21 @@ THttpServerConn::THttpServerConn(const TSocket& s) } THttpServerConn::THttpServerConn(const TSocket& s, size_t outputBufferSize) - : Impl_(new TImpl(s, outputBufferSize)) + : THttpServerConn(MakeHolder<TBlockingSocketStreams>(s), outputBufferSize) { } -THttpServerConn::~THttpServerConn() { -} - -THttpInput* THttpServerConn::Input() noexcept { - return Impl_->Input(); +THttpServerConn::THttpServerConn(THolder<ISocketStreams> socketStreams, size_t outputBufferSize) + : SocketStreams_(std::move(socketStreams)) + , BufferedOutput_(SocketStreams_->Output(), outputBufferSize) + , HttpInput_(SocketStreams_->Input()) + , HttpOutput_(&BufferedOutput_, &HttpInput_) +{ } -THttpOutput* THttpServerConn::Output() noexcept { - return Impl_->Output(); +THttpServerConn::~THttpServerConn() { } void THttpServerConn::Reset() { - return Impl_->Reset(); + return SocketStreams_->Reset(); } diff --git a/library/cpp/http/server/conn.h b/library/cpp/http/server/conn.h index 3aa5329af42..62a3e86ac95 100644 --- a/library/cpp/http/server/conn.h +++ b/library/cpp/http/server/conn.h @@ -2,18 +2,37 @@ #include <library/cpp/http/io/stream.h> #include <util/generic/ptr.h> +#include <util/stream/buffered.h> +class IInputStream; +class IOutputStream; class TSocket; /// Потоки ввода/вывода для получения запросов и отправки ответов HTTP-сервера. class THttpServerConn { public: - explicit THttpServerConn(const TSocket& s); + class ISocketStreams { + public: + virtual ~ISocketStreams() {} + + virtual IInputStream* Input() = 0; + virtual IOutputStream* Output() = 0; + virtual void Reset() = 0; + }; +public: + THttpServerConn(const TSocket& s); THttpServerConn(const TSocket& s, size_t outputBufferSize); + THttpServerConn(THolder<ISocketStreams> socketStreams, size_t outputBufferSize); + ~THttpServerConn(); - THttpInput* Input() noexcept; - THttpOutput* Output() noexcept; + THttpInput* Input() noexcept { + return &HttpInput_; + } + + THttpOutput* Output() noexcept { + return &HttpOutput_; + } inline const THttpInput* Input() const noexcept { return const_cast<THttpServerConn*>(this)->Input(); @@ -30,8 +49,10 @@ public: } void Reset(); - private: - class TImpl; - THolder<TImpl> Impl_; + THolder<ISocketStreams> SocketStreams_; + + TBufferedOutput BufferedOutput_; + THttpInput HttpInput_; + THttpOutput HttpOutput_; }; diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp index 13088f5e442..79a1194c770 100644 --- a/library/cpp/http/server/http.cpp +++ b/library/cpp/http/server/http.cpp @@ -725,6 +725,10 @@ void TClientRequest::ResetConnection() { } } +THolder<THttpServerConn> TClientRequest::CreateHttpConnection(const TSocket& s, size_t outputBufferSize) { + return MakeHolder<THttpServerConn>(s, outputBufferSize); +} + void TClientRequest::Process(void* ThreadSpecificResource) { THolder<TClientRequest> this_(this); @@ -733,11 +737,7 @@ void TClientRequest::Process(void* ThreadSpecificResource) { try { if (!HttpConn_) { const size_t outputBufferSize = HttpServ()->Options().OutputBufferSize; - if (outputBufferSize) { - HttpConn_.Reset(new THttpServerConn(Socket(), outputBufferSize)); - } else { - HttpConn_.Reset(new THttpServerConn(Socket())); - } + HttpConn_ = CreateHttpConnection(Socket(), outputBufferSize ? outputBufferSize : Socket().MaximumTransferUnit()); auto maxRequestsPerConnection = HttpServ()->Options().MaxRequestsPerConnection; HttpConn_->Output()->EnableKeepAlive(HttpServ()->Options().KeepAliveEnabled && (!maxRequestsPerConnection || Conn_->ReceivedRequests < maxRequestsPerConnection)); diff --git a/library/cpp/http/server/http.h b/library/cpp/http/server/http.h index 62dfd5a50fa..5b23cb21a00 100644 --- a/library/cpp/http/server/http.h +++ b/library/cpp/http/server/http.h @@ -141,6 +141,8 @@ private: Y_UNUSED(ThreadSpecificResource); return true; } + virtual THolder<THttpServerConn> CreateHttpConnection(const TSocket& s, size_t outputBuffer); + void Process(void* ThreadSpecificResource) override; public: diff --git a/library/cpp/http/server/http_ut.cpp b/library/cpp/http/server/http_ut.cpp index 0fc44fa5983..b3280745929 100644 --- a/library/cpp/http/server/http_ut.cpp +++ b/library/cpp/http/server/http_ut.cpp @@ -7,6 +7,7 @@ #include <util/generic/cast.h> #include <util/stream/output.h> #include <util/stream/zlib.h> +#include <util/stream/tee.h> #include <util/system/datetime.h> #include <util/system/mutex.h> #include <util/random/random.h> @@ -1016,4 +1017,112 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { t2->Join(); UNIT_ASSERT_EQUAL_C(results, (THashSet<TString>({"Zoooo", "TTL Exceed"})), "Results is {" + ToString(results) + "}"); } + + Y_UNIT_TEST(TestCustomSocketStreams) { + // Check that TClientRequest::CreateHttpConnection override works + + class TCustomSocketStreamsServer: public THttpServer::ICallBack { + public: + TStringStream InputCopy; + TStringStream OutputCopy; + private: + class TTeeInput + : public IInputStream + { + public: + TTeeInput(IInputStream* s, IOutputStream* copy) + : S_(s) + , Copy_(copy) + { + } + + size_t DoRead(void* buf, size_t len) override { + void* begin = buf; + size_t res = S_->Read(buf, len); + Copy_->Write(begin, res); + return res; + } + + private: + IInputStream* const S_; + IOutputStream* const Copy_; + }; + + class TBlockingSocketStreams: public THttpServerConn::ISocketStreams { + public: + explicit TBlockingSocketStreams(const TSocket& s, TCustomSocketStreamsServer* server) + : Socket_(s) + , Input_(Socket_) + , TI_(&Input_, &server->InputCopy) + , Output_(Socket_) + , TO_(&Output_, &server->OutputCopy) + { + } + + IInputStream* Input() override { + return &TI_; + } + + IOutputStream* Output() override { + return &TO_; + } + + void Reset() override {} + private: + TSocket Socket_; + TSocketInput Input_; + TTeeInput TI_; + TSocketOutput Output_; + TTeeOutput TO_; + }; + + class TRequest: public TClientRequest { + public: + TRequest(TCustomSocketStreamsServer* server) + : Server_(server) + { + } + + bool Reply(void* /*tsr*/) override { + Output() << "HTTP/1.1 200 Ok\r\n\r\n"; + Output().Finish(); + return true; + } + + THolder<THttpServerConn> CreateHttpConnection(const TSocket& s, size_t outputBuffer) override { + return MakeHolder<THttpServerConn>(MakeHolder<TBlockingSocketStreams>(s, Server_), outputBuffer); + } + private: + TCustomSocketStreamsServer* const Server_; + }; + + public: + TClientRequest* CreateClient() override { + return new TRequest(this); + } + + void OnException() override { + ExceptionMessage = CurrentExceptionMessage(); + } + + TString ExceptionMessage; + }; + + TPortManager portManager; + const ui16 port = portManager.GetPort(); + TCustomSocketStreamsServer server; + THttpServer::TOptions options(port); + options.nThreads = 1; + options.MaxConnections = 2; + THttpServer srv(&server, options); + + UNIT_ASSERT(srv.Start()); + + TSocket socket(TNetworkAddress("localhost", port), TDuration::Seconds(10)); + + SendRequest(socket, port); + + UNIT_ASSERT_STRINGS_EQUAL(server.InputCopy.Str(), TStringBuilder() << "GET / HTTP/1.1\r\nHost: localhost:" << port << "\r\nConnection: Keep-Alive\r\n\r\n"); + UNIT_ASSERT_STRINGS_EQUAL(server.OutputCopy.Str(), TStringBuilder() << "HTTP/1.1 200 Ok\r\nConnection: Keep-Alive\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n"); + } } diff --git a/library/cpp/protobuf/json/proto/ya.make b/library/cpp/protobuf/json/proto/ya.make index b565c78d160..fd9e0563bc8 100644 --- a/library/cpp/protobuf/json/proto/ya.make +++ b/library/cpp/protobuf/json/proto/ya.make @@ -5,4 +5,6 @@ SRCS( enum_options.proto ) +INCLUDE_TAGS(DOCS_PROTO) + END() diff --git a/library/cpp/streams/lzma/lzma_ut.cpp b/library/cpp/streams/lzma/lzma_ut.cpp index 847e98d1caa..dbcd7d78fac 100644 --- a/library/cpp/streams/lzma/lzma_ut.cpp +++ b/library/cpp/streams/lzma/lzma_ut.cpp @@ -10,7 +10,7 @@ class TStrokaByOneByte: public IZeroCopyInput { public: TStrokaByOneByte(const TString& s) : Data(s) - , Pos(s.data()) + , Pos(Data.data()) { } diff --git a/library/cpp/streams/lzma/ut/ya.make b/library/cpp/streams/lzma/ut/ya.make index 99b7e5e401a..88fdb91dd1b 100644 --- a/library/cpp/streams/lzma/ut/ya.make +++ b/library/cpp/streams/lzma/ut/ya.make @@ -1,5 +1,7 @@ UNITTEST_FOR(library/cpp/streams/lzma) +NO_COW() + SRCS( lzma_ut.cpp ) diff --git a/library/cpp/yt/memory/ref_counted-inl.h b/library/cpp/yt/memory/ref_counted-inl.h index 3d5b212bf96..94acfc6669b 100644 --- a/library/cpp/yt/memory/ref_counted-inl.h +++ b/library/cpp/yt/memory/ref_counted-inl.h @@ -8,12 +8,48 @@ #include <util/system/sanitizers.h> +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE +#include <util/system/types.h> +#endif + #include <stdlib.h> namespace NYT { //////////////////////////////////////////////////////////////////////////////// +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + +namespace NDetail { + +// A distinctive marker held by a live signature (after address salting). +constexpr ui64 RefCountedAliveSignatureMagic = 0xa11e0b1ec70a11e0ULL; + +// A distinct poison written into the signature once the object is destroyed. +constexpr ui64 RefCountedDeadSignatureMagic = 0xdeadbeefdeadbeefULL; + +//! The alive signature for a signature word living at #address: the magic XOR-ed +//! with the word's own address. +//! +//! The signature is embedded as the first word of every TRefCounter: the ctor +//! stamps ComputeRefCountedAliveSignature(&signature) and strong-death overwrites +//! it with RefCountedDeadSignatureMagic. A coredump walker tells a live object +//! from freed-but-unreclaimed memory: for a word at address S, +//! *S == ComputeRefCountedAliveSignature(S) means alive, +//! *S == RefCountedDeadSignatureMagic means freed. The address salt keeps a +//! signature from validating if copied verbatim elsewhere (it carries the old +//! address) and makes a chance match of arbitrary memory a ~2^-64 event. +Y_FORCE_INLINE ui64 ComputeRefCountedAliveSignature(const void* address) noexcept +{ + return RefCountedAliveSignatureMagic ^ reinterpret_cast<uintptr_t>(address); +} + +} // namespace NDetail + +#endif + +//////////////////////////////////////////////////////////////////////////////// + // TODO(babenko): move to hazard pointers void RetireHazardPointer( void* protectedPtr, @@ -190,6 +226,19 @@ Y_FORCE_INLINE int TRefCounter::GetRefCount() const noexcept return StrongCount_.load(std::memory_order::acquire); } +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + +Y_FORCE_INLINE TRefCounter::TRefCounter() noexcept + : Signature_(NDetail::ComputeRefCountedAliveSignature(this)) +{ } + +Y_FORCE_INLINE ui64 TRefCounter::GetSignature() const noexcept +{ + return Signature_; +} + +#endif + Y_FORCE_INLINE void TRefCounter::Ref(int n) const noexcept { YT_ASSERT(n >= 0); @@ -238,6 +287,11 @@ Y_FORCE_INLINE bool TRefCounter::Unref(int n) const if (oldStrongCount == n) { std::atomic_thread_fence(std::memory_order::acquire); NSan::Acquire(&StrongCount_); +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + // Last strong ref gone: the object is about to be destroyed. Poison now so + // a core walker sees freed-but-unreclaimed memory as dead. + Signature_ = NDetail::RefCountedDeadSignatureMagic; +#endif return true; } else { return false; diff --git a/library/cpp/yt/memory/ref_counted.h b/library/cpp/yt/memory/ref_counted.h index 154bbd14659..e82d22936e4 100644 --- a/library/cpp/yt/memory/ref_counted.h +++ b/library/cpp/yt/memory/ref_counted.h @@ -39,6 +39,10 @@ private: class TRefCounter { public: +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + TRefCounter() noexcept; +#endif + //! Returns current number of strong references to the object. /*! * Note that you should never ever use this method in production code. @@ -69,7 +73,22 @@ public: //! Decrements the weak reference counter. bool WeakUnref() const; +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + //! Returns the raw signature word. Used only by unit tests; the coredump + //! walker reads the word directly from memory. + ui64 GetSignature() const noexcept; +#endif + private: +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + // An address-salted liveness marker. NB: Must be the first member -- the + // salt is its own address and a core validator keys off its offset. Stamped + // alive by the ctor (see -inl); poisoned at strong-death (see Unref). + // TRefCounter is already non-copyable (atomics below), so the salted value + // can't be copied astray. + mutable ui64 Signature_; +#endif + // NB: Must be 64 bit as TAtomicIntrusivePtr grabs refs in 64K batches. using TRefCount = i64; mutable std::atomic<TRefCount> StrongCount_ = 1; diff --git a/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp b/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp index 2b665b41473..86b6341f915 100644 --- a/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp +++ b/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp @@ -691,5 +691,43 @@ TEST(TIntrusivePtrTest, HeterogeneousLookup) //////////////////////////////////////////////////////////////////////////////// +#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE + +TEST(TRefCountedSignatureTest, FinalType) +{ + // Final, non-derived type: New lays a TRefCounter right before the object. + struct TObject final + { + ui64 Data = 0; + }; + + auto obj = New<TObject>(); + const auto* refCounter = GetRefCounter(obj.Get()); + + // The signature is salted purely by the counter's own address. + EXPECT_EQ(refCounter->GetSignature(), NDetail::ComputeRefCountedAliveSignature(refCounter)); + EXPECT_NE(refCounter->GetSignature(), NDetail::RefCountedDeadSignatureMagic); +} + +TEST(TRefCountedSignatureTest, DerivedType) +{ + // Polymorphic (TRefCounted-derived) type: the counter is a base subobject. + struct TObject + : public TRefCounted + { + ui64 Data = 0; + }; + + auto obj = New<TObject>(); + const auto* refCounter = GetRefCounter(obj.Get()); + + EXPECT_EQ(refCounter->GetSignature(), NDetail::ComputeRefCountedAliveSignature(refCounter)); + EXPECT_NE(refCounter->GetSignature(), NDetail::RefCountedDeadSignatureMagic); +} + +#endif + +//////////////////////////////////////////////////////////////////////////////// + } // namespace } // namespace NYT diff --git a/library/cpp/yt/misc/port.h b/library/cpp/yt/misc/port.h index e54a2450ce9..b7b350f4581 100644 --- a/library/cpp/yt/misc/port.h +++ b/library/cpp/yt/misc/port.h @@ -8,6 +8,13 @@ #define YT_ENABLE_REF_COUNTED_TRACKING #endif +#if !defined(NDEBUG) && !defined(YT_DISABLE_REF_COUNTED_SIGNATURE) + // This define embeds a salted, poisonable signature into every TRefCounter + // (see ref_counted.h). It lets a coredump walker tell a live ref-counted object + // from freed-but-unreclaimed memory. + #define YT_ENABLE_REF_COUNTED_SIGNATURE +#endif + // This define enables logging with TRACE level. You can still disable trace logging // for particular TU by discarding this macro identifier. #define YT_ENABLE_TRACE_LOGGING diff --git a/library/cpp/yt/string/string.cpp b/library/cpp/yt/string/string.cpp index 1da50b4ccdd..e4a2afeca5e 100644 --- a/library/cpp/yt/string/string.cpp +++ b/library/cpp/yt/string/string.cpp @@ -65,6 +65,26 @@ TString CamelCaseToUnderscoreCase(TStringBuf str) //////////////////////////////////////////////////////////////////////////////// +std::string AsciiStringToLower(TStringBuf value) +{ + std::string result(value.size(), '\0'); + for (size_t index = 0; index < value.size(); ++index) { + result[index] = ::AsciiToLower(value[index]); + } + return result; +} + +std::string AsciiStringToUpper(TStringBuf value) +{ + std::string result(value.size(), '\0'); + for (size_t index = 0; index < value.size(); ++index) { + result[index] = ::AsciiToUpper(value[index]); + } + return result; +} + +//////////////////////////////////////////////////////////////////////////////// + [[nodiscard]] TStringBuf TrimLeadingWhitespaces(TStringBuf str Y_LIFETIME_BOUND) { auto begin = str.find_first_not_of(' '); @@ -292,7 +312,7 @@ size_t TCaseInsensitiveStringHasher::operator()(TStringBuf arg) const { auto compute = [&] (char* buffer) { for (size_t index = 0; index < arg.length(); ++index) { - buffer[index] = AsciiToLower(arg[index]); + buffer[index] = ::AsciiToLower(arg[index]); } return ComputeHash(TStringBuf(buffer, arg.length())); }; diff --git a/library/cpp/yt/string/string.h b/library/cpp/yt/string/string.h index 346ec04ef4d..968ac3a0554 100644 --- a/library/cpp/yt/string/string.h +++ b/library/cpp/yt/string/string.h @@ -148,6 +148,15 @@ TString UnderscoreCaseToCamelCase(TStringBuf str); void CamelCaseToUnderscoreCase(TStringBuilderBase* builder, TStringBuf str); TString CamelCaseToUnderscoreCase(TStringBuf str); +//! ASCII case conversion returning |std::string|. +//! TODO(babenko): likely a temporary workaround; these exist only because util's +//! |to_lower|/|to_upper| are |TString|-based (forcing a |TString| round-trip for +//! |std::string| callers). Drop once util gains |std::string| support. +//! Returns a copy of |value| with every ASCII letter lowercased; non-ASCII bytes are left intact. +std::string AsciiStringToLower(TStringBuf value); +//! Returns a copy of |value| with every ASCII letter uppercased; non-ASCII bytes are left intact. +std::string AsciiStringToUpper(TStringBuf value); + [[nodiscard]] TStringBuf TrimLeadingWhitespaces(TStringBuf str Y_LIFETIME_BOUND); [[nodiscard]] TStringBuf Trim(TStringBuf str Y_LIFETIME_BOUND, TStringBuf whitespaces = " "); diff --git a/library/cpp/yt/threading/rw_spin_lock.cpp b/library/cpp/yt/threading/rw_spin_lock.cpp index efc0e90167a..3150c1aa132 100644 --- a/library/cpp/yt/threading/rw_spin_lock.cpp +++ b/library/cpp/yt/threading/rw_spin_lock.cpp @@ -41,7 +41,8 @@ void TUncheckedReaderWriterSpinLock::AcquireWriterSlow() noexcept namespace { -// NB: Reading this after destruction is UB, but we can't deal with Static Initialization Order Fiasco in any other feasible way :( +// NB: Reading this after destruction is UB, but we can't deal with +// Static Initialization Order Fiasco in any other feasible way :( YT_DEFINE_THREAD_LOCAL(bool, ThreadLockTrackerDestroyed, false); class TThreadLockTracker @@ -61,13 +62,17 @@ public: ~TThreadLockTracker() { ThreadLockTrackerDestroyed() = true; - }; + } private: THashSet<TCheckedReaderWriterSpinLock*> ThreadLocksAcquired_; }; -YT_DEFINE_THREAD_LOCAL(TThreadLockTracker, ThreadLockTracker); +YT_PREVENT_TLS_CACHING static TThreadLockTracker& ThreadLockTracker() +{ + static thread_local TThreadLockTracker Instance; + return Instance; +} } // namespace |
