summaryrefslogtreecommitdiffstats
path: root/library/cpp
diff options
context:
space:
mode:
authorYDBot <[email protected]>2026-07-02 18:29:58 +0000
committerYDBot <[email protected]>2026-07-02 18:29:58 +0000
commita52ffb856a584a5099e37b9c060da3ba4a3db5fb (patch)
tree59c7f09bdc9e52d11ce6ddf8b6fabdfe1e73a8d6 /library/cpp
parent6faa15e1fbe202f0fddb7c1ffe9886dc81a0425f (diff)
parentd5cfc51dd97284c59104c50bb156d1a8d9a09dac (diff)
Merge pull request #45225 from ydb-platform/merge-rightlib-260701-1735
Diffstat (limited to 'library/cpp')
-rw-r--r--library/cpp/charset/ut/ya.make2
-rw-r--r--library/cpp/charset/wide_ut.cpp8
-rw-r--r--library/cpp/cron_expression/cron_expression.cpp5
-rw-r--r--library/cpp/cron_expression/ut/cron_expression_ut.cpp86
-rw-r--r--library/cpp/dot_product/common.h6
-rw-r--r--library/cpp/dot_product/dot_product.cpp19
-rw-r--r--library/cpp/dot_product/dot_product.h30
-rw-r--r--library/cpp/dot_product/dot_product_avx2.cpp226
-rw-r--r--library/cpp/dot_product/dot_product_avx2.h13
-rw-r--r--library/cpp/dot_product/dot_product_simple.cpp40
-rw-r--r--library/cpp/dot_product/dot_product_simple.h13
-rw-r--r--library/cpp/dot_product/dot_product_sse.cpp163
-rw-r--r--library/cpp/dot_product/dot_product_sse.h13
-rw-r--r--library/cpp/dot_product/dot_product_ut.cpp83
-rw-r--r--library/cpp/http/server/conn.cpp81
-rw-r--r--library/cpp/http/server/conn.h33
-rw-r--r--library/cpp/http/server/http.cpp10
-rw-r--r--library/cpp/http/server/http.h2
-rw-r--r--library/cpp/http/server/http_ut.cpp109
-rw-r--r--library/cpp/protobuf/json/proto/ya.make2
-rw-r--r--library/cpp/streams/lzma/lzma_ut.cpp2
-rw-r--r--library/cpp/streams/lzma/ut/ya.make2
-rw-r--r--library/cpp/yt/memory/ref_counted-inl.h54
-rw-r--r--library/cpp/yt/memory/ref_counted.h19
-rw-r--r--library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp38
-rw-r--r--library/cpp/yt/misc/port.h7
-rw-r--r--library/cpp/yt/string/string.cpp22
-rw-r--r--library/cpp/yt/string/string.h9
-rw-r--r--library/cpp/yt/threading/rw_spin_lock.cpp11
29 files changed, 1042 insertions, 66 deletions
diff --git a/library/cpp/charset/ut/ya.make b/library/cpp/charset/ut/ya.make
index c22436f8acf..62f3426f67c 100644
--- a/library/cpp/charset/ut/ya.make
+++ b/library/cpp/charset/ut/ya.make
@@ -1,5 +1,7 @@
UNITTEST_FOR(library/cpp/charset)
+NO_COW()
+
SRCS(
iconv_ut.cpp
recyr_int_ut.cpp
diff --git a/library/cpp/charset/wide_ut.cpp b/library/cpp/charset/wide_ut.cpp
index 93567161bab..4f5a666cb8e 100644
--- a/library/cpp/charset/wide_ut.cpp
+++ b/library/cpp/charset/wide_ut.cpp
@@ -10,6 +10,8 @@
#include <algorithm>
+extern const int TStringUseCow;
+
namespace {
//! three UTF8 encoded russian letters (A, B, V)
const char yandexCyrillicAlphabet[] =
@@ -264,10 +266,12 @@ void TConversionTest::TestRecodeIntoString() {
TUtf16String copy = sUnicode; // increase ref-counter
wres = NDetail::Recode<char>(UTF8Text, sUnicode, CODES_UTF8);
UNIT_ASSERT(sUnicode == UnicodeText); // same content
+ if (TStringUseCow) {
#ifndef TSTRING_IS_STD_STRING
- UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied)
- UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer
+ UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied)
+ UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer
#endif
+ }
UNIT_ASSERT(sUnicode.size() == wres.size()); // same content
}
diff --git a/library/cpp/cron_expression/cron_expression.cpp b/library/cpp/cron_expression/cron_expression.cpp
index 55b64ee6d24..31a538a0766 100644
--- a/library/cpp/cron_expression/cron_expression.cpp
+++ b/library/cpp/cron_expression/cron_expression.cpp
@@ -1060,12 +1060,15 @@ private:
}
value = GetField(calendar, ECronField::CF_DAY_OF_MONTH);
+ // FindDay may cross a month/year boundary keeping the same day number (e.g. 'L-1'), check them too
+ const auto prevMonth = calendar.month();
+ const auto prevYear = calendar.year();
updateValue = FindDay(calendar, Target_.GetDayInMonth(), value, GetWeekday(calendar), Target_.GetFlags(), resets, offset);
if (updateValue.Empty()) {
ythrow yexception() << ErrorDateNotExists;
}
- if (value == updateValue) {
+ if (value == updateValue && prevMonth == calendar.month() && prevYear == calendar.year()) {
resets.Set(static_cast<uint32_t>(ECronField::CF_DAY_OF_MONTH));
} else {
continue;
diff --git a/library/cpp/cron_expression/ut/cron_expression_ut.cpp b/library/cpp/cron_expression/ut/cron_expression_ut.cpp
index 999f1e00260..0a423dd85bd 100644
--- a/library/cpp/cron_expression/ut/cron_expression_ut.cpp
+++ b/library/cpp/cron_expression/ut/cron_expression_ut.cpp
@@ -349,6 +349,69 @@ Y_UNIT_TEST_SUITE(TestNext) {
TestNext("@minutely", "2024-04-22T16:05:32Z", "2024-04-22T16:06:00Z");
TestNext("@secondly", "2021-12-07T12:00:00Z", "2021-12-07T12:00:01Z");
}
+
+ Y_UNIT_TEST(RangesListsAndSteps) {
+ TestNext("0 0 9-17 * * *", "2021-03-15T08:00:00Z", "2021-03-15T09:00:00Z");
+ TestNext("0 0 9-17 * * *", "2021-03-15T20:00:00Z", "2021-03-16T09:00:00Z");
+ TestNext("0 0,30 * * * *", "2021-03-15T10:05:00Z", "2021-03-15T10:30:00Z");
+ TestNext("0 */15 * * * *", "2021-03-15T10:07:00Z", "2021-03-15T10:15:00Z");
+ TestNext("0 0 */6 * * *", "2021-03-15T05:00:00Z", "2021-03-15T06:00:00Z");
+ TestNext("0 0 0 1,15 * *", "2021-03-16T00:00:00Z", "2021-04-01T00:00:00Z");
+ TestNext("0 0 0 1-7 * *", "2021-03-10T00:00:00Z", "2021-04-01T00:00:00Z");
+ TestNext("0 0 0 */10 * *", "2021-03-05T00:00:00Z", "2021-03-11T00:00:00Z");
+ TestNext("0 30 9 * * *", "2021-03-15T10:20:30Z", "2021-03-16T09:30:00Z");
+ TestNext("0 0 * * * *", "2021-03-15T10:20:30Z", "2021-03-15T11:00:00Z");
+ TestNext("15 10 * * *", "2021-03-15T10:20:30Z", "2021-03-16T10:15:00Z");
+ }
+
+ Y_UNIT_TEST(MonthsAndNames) {
+ TestNext("0 0 0 1 1 *", "2021-06-15T00:00:00Z", "2022-01-01T00:00:00Z");
+ TestNext("0 0 0 1 JAN *", "2021-06-15T00:00:00Z", "2022-01-01T00:00:00Z");
+ TestNext("0 0 0 1 2-4 *", "2021-06-15T00:00:00Z", "2022-02-01T00:00:00Z");
+ TestNext("0 0 0 1 DEC *", "2021-12-31T00:00:00Z", "2022-12-01T00:00:00Z");
+ TestNext("0 0 0 29 2 *", "2021-03-01T00:00:00Z", "2024-02-29T00:00:00Z");
+ }
+
+ Y_UNIT_TEST(DaysOfWeek) {
+ TestNext("0 0 0 * * 1", "2021-03-15T12:00:00Z", "2021-03-22T00:00:00Z");
+ TestNext("0 0 0 * * MON", "2021-03-15T12:00:00Z", "2021-03-22T00:00:00Z");
+ TestNext("0 0 0 * * 0", "2021-03-15T12:00:00Z", "2021-03-21T00:00:00Z");
+ TestNext("0 0 0 * * 7", "2021-03-15T12:00:00Z", "2021-03-21T00:00:00Z");
+ TestNext("0 0 0 * * 6-7", "2021-03-15T12:00:00Z", "2021-03-20T00:00:00Z");
+ TestNext("0 0 0 * * MON-FRI", "2021-03-19T12:00:00Z", "2021-03-22T00:00:00Z");
+ TestNext("0 0 0 * * SAT,SUN", "2021-03-15T12:00:00Z", "2021-03-20T00:00:00Z");
+ TestNext("0 0 12 * * 5", "2021-03-15T12:00:00Z", "2021-03-19T12:00:00Z");
+ }
+
+ Y_UNIT_TEST(NthWeekdayOfMonth) {
+ TestNext("0 0 0 * * 1#1", "2021-03-01T00:00:00Z", "2021-04-05T00:00:00Z");
+ TestNext("0 0 0 * * 1#2", "2021-03-01T00:00:00Z", "2021-03-08T00:00:00Z");
+ TestNext("0 0 0 * * 5#3", "2021-03-01T00:00:00Z", "2021-03-19T00:00:00Z");
+ TestNext("0 0 0 * * 1#-1", "2021-03-01T00:00:00Z", "2021-03-29T00:00:00Z");
+ TestNext("0 0 0 * * 3#-2", "2021-03-01T00:00:00Z", "2021-03-24T00:00:00Z");
+ }
+
+ Y_UNIT_TEST(LastDaysOfMonthWithTime) {
+ // 'L-N' rolling into the next month with an unchanged day number must keep the requested time
+ TestNext("59 59 23 L-1 * *", "2026-06-30T03:14:00Z", "2026-07-30T23:59:59Z");
+ TestNext("59 59 23 L-1 * *", "2026-09-30T10:00:00Z", "2026-10-30T23:59:59Z");
+ TestNext("59 59 23 L-1 * *", "2026-11-30T10:00:00Z", "2026-12-30T23:59:59Z");
+ TestNext("59 59 23 L-1 * *", "2026-04-30T10:00:00Z", "2026-05-30T23:59:59Z");
+ TestNext("59 59 23 L-1 * *", "2026-07-01T00:00:00Z", "2026-07-30T23:59:59Z");
+ TestNext("0 30 12 L-2 * *", "2026-06-29T13:00:00Z", "2026-07-29T12:30:00Z");
+ TestNext("59 59 23 L * *", "2021-03-31T10:00:00Z", "2021-03-31T23:59:59Z");
+ TestNext("59 59 23 L * *", "2021-04-30T23:59:59Z", "2021-05-31T23:59:59Z");
+ TestNext("59 59 23 L * *", "2025-08-31T05:00:00Z", "2025-08-31T23:59:59Z");
+ TestNext("0 0 0 L * *", "2021-02-10T00:00:00Z", "2021-02-28T00:00:00Z");
+ TestNext("0 0 0 L * *", "2020-02-10T00:00:00Z", "2020-02-29T00:00:00Z");
+ TestNext("0 0 0 L-5 * *", "2021-02-10T00:00:00Z", "2021-02-23T00:00:00Z");
+ TestNext("30 0 0 L-1 * *", "2025-08-30T00:00:00Z", "2025-08-30T00:00:30Z");
+ }
+
+ Y_UNIT_TEST(YearField) {
+ TestNext("0 0 0 1 1 * 2030", "2021-01-01T00:00:00Z", "2030-01-01T00:00:00Z");
+ TestNext("0 0 0 29 2 * 2024", "2021-01-01T00:00:00Z", "2024-02-29T00:00:00Z");
+ }
}
Y_UNIT_TEST_SUITE(TestPrev) {
@@ -441,6 +504,29 @@ Y_UNIT_TEST_SUITE(TestPrev) {
TestPrev("30 50 23 20,21,22 * *", "2023-07-22T12:34:56Z", "2023-07-21T23:50:30Z");
TestPrev("30 50 23 20,21,22 * *", "2023-07-23T12:34:56Z", "2023-07-22T23:50:30Z");
}
+
+ Y_UNIT_TEST(MoreCases) {
+ TestPrev("0 0 0 * * *", "2021-03-15T10:20:30Z", "2021-03-15T00:00:00Z");
+ TestPrev("0 0 9-17 * * *", "2021-03-15T08:00:00Z", "2021-03-14T17:00:00Z");
+ TestPrev("0 0,30 * * * *", "2021-03-15T10:05:00Z", "2021-03-15T10:00:00Z");
+ TestPrev("0 0 0 1 1 *", "2021-06-15T00:00:00Z", "2021-01-01T00:00:00Z");
+ TestPrev("0 0 0 29 2 *", "2021-03-01T00:00:00Z", "2020-02-29T00:00:00Z");
+ TestPrev("0 0 0 * * MON", "2021-03-15T12:00:00Z", "2021-03-15T00:00:00Z");
+ TestPrev("0 0 0 * * SAT,SUN", "2021-03-15T12:00:00Z", "2021-03-14T00:00:00Z");
+ TestPrev("0 0 0 * * 1#-1", "2021-03-01T00:00:00Z", "2021-02-22T00:00:00Z");
+ TestPrev("0 0 0 L * *", "2021-02-10T00:00:00Z", "2021-01-31T00:00:00Z");
+ TestPrev("0 0 0 LW * *", "2021-05-10T00:00:00Z", "2021-04-30T00:00:00Z");
+ TestPrev("0 0 0 15W * *", "2021-05-01T00:00:00Z", "2021-04-15T00:00:00Z");
+ }
+
+ Y_UNIT_TEST(LastDaysOfMonthWithTime) {
+ // mirror of the TestNext case in the prev direction
+ TestPrev("59 59 23 L * *", "2025-08-31T05:00:00Z", "2025-07-31T23:59:59Z");
+ TestPrev("30 0 0 L-1 * *", "2025-08-30T00:00:00Z", "2025-07-30T00:00:30Z");
+ TestPrev("30 0 0 L-1 * *", "2025-10-30T00:00:00Z", "2025-09-29T00:00:30Z");
+ TestPrev("59 59 23 L-1 * *", "2026-06-30T03:14:00Z", "2026-06-29T23:59:59Z");
+ TestPrev("0 30 12 L-2 * *", "2026-06-29T13:00:00Z", "2026-06-28T12:30:00Z");
+ }
}
Y_UNIT_TEST_SUITE(AliceBasicUsageExamples) {
diff --git a/library/cpp/dot_product/common.h b/library/cpp/dot_product/common.h
index 25cd6153404..bd72a6ed3df 100644
--- a/library/cpp/dot_product/common.h
+++ b/library/cpp/dot_product/common.h
@@ -18,3 +18,9 @@ enum class ETriWayDotProductComputeMask: unsigned {
Left = 0b110, // skip computation of R·R
Right = 0b011, // skip computation of L·L
};
+
+struct TTriWayDotProductFloatI8 {
+ float LL = 0;
+ float LR = 0;
+ float RR = 0;
+};
diff --git a/library/cpp/dot_product/dot_product.cpp b/library/cpp/dot_product/dot_product.cpp
index 79ccbdb3b7d..003dbe88335 100644
--- a/library/cpp/dot_product/dot_product.cpp
+++ b/library/cpp/dot_product/dot_product.cpp
@@ -15,11 +15,18 @@ namespace NDotProductImpl {
ui32 (*DotProductUi8Impl)(const ui8* lhs, const ui8* rhs, size_t length) noexcept = &DotProductSimple;
i64 (*DotProductI32Impl)(const i32* lhs, const i32* rhs, size_t length) noexcept = &DotProductSimple;
float (*DotProductFloatImpl)(const float* lhs, const float* rhs, size_t length) noexcept = &DotProductSimple;
+ float (*DotProductFloatI8Impl)(const float* lhs, const i8* rhs, size_t length) noexcept = &DotProductSimple;
double (*DotProductDoubleImpl)(const double* lhs, const double* rhs, size_t length) noexcept = &DotProductSimple;
TTriWayDotProduct<float> (*TriWayDotProductImpl)
(const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept = &TriWayDotProductSimple;
+ TTriWayDotProductFloatI8 (*TriWayDotProductFloatI8Impl)
+ (const float* lhs, const i8* rhs, size_t length) noexcept = &TriWayDotProductFloatI8Simple;
+
+ TTriWayDotProduct<i32> (*TriWayDotProductI8Impl)
+ (const i8* lhs, const i8* rhs, size_t length) noexcept = &TriWayDotProductI8Simple;
+
namespace {
[[maybe_unused]] const int _ = [] {
@@ -28,16 +35,22 @@ namespace NDotProductImpl {
DotProductUi8Impl = &DotProductAvx2;
DotProductI32Impl = &DotProductAvx2;
DotProductFloatImpl = &DotProductAvx2;
+ DotProductFloatI8Impl = &DotProductFloatI8Avx2;
DotProductDoubleImpl = &DotProductAvx2;
TriWayDotProductImpl = &TriWayDotProductAvx2;
+ TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Avx2;
+ TriWayDotProductI8Impl = &TriWayDotProductI8Avx2;
} else {
#ifdef ARCADIA_SSE
DotProductI8Impl = &DotProductSse;
DotProductUi8Impl = &DotProductSse;
DotProductI32Impl = &DotProductSse;
DotProductFloatImpl = &DotProductSse;
+ DotProductFloatI8Impl = &DotProductSse;
DotProductDoubleImpl = &DotProductSse;
TriWayDotProductImpl = &TriWayDotProductSse;
+ TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Sse;
+ TriWayDotProductI8Impl = &TriWayDotProductI8Sse;
#endif
}
return 0;
@@ -171,15 +184,21 @@ namespace NDotProduct {
NDotProductImpl::DotProductUi8Impl = &DotProductSse;
NDotProductImpl::DotProductI32Impl = &DotProductSse;
NDotProductImpl::DotProductFloatImpl = &DotProductSse;
+ NDotProductImpl::DotProductFloatI8Impl = &DotProductSse;
NDotProductImpl::DotProductDoubleImpl = &DotProductSse;
NDotProductImpl::TriWayDotProductImpl = &TriWayDotProductSse;
+ NDotProductImpl::TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Sse;
+ NDotProductImpl::TriWayDotProductI8Impl = &TriWayDotProductI8Sse;
#else
NDotProductImpl::DotProductI8Impl = &DotProductSimple;
NDotProductImpl::DotProductUi8Impl = &DotProductSimple;
NDotProductImpl::DotProductI32Impl = &DotProductSimple;
NDotProductImpl::DotProductFloatImpl = &DotProductSimple;
+ NDotProductImpl::DotProductFloatI8Impl = &DotProductSimple;
NDotProductImpl::DotProductDoubleImpl = &DotProductSimple;
NDotProductImpl::TriWayDotProductImpl = &TriWayDotProductSimple;
+ NDotProductImpl::TriWayDotProductFloatI8Impl = &TriWayDotProductFloatI8Simple;
+ NDotProductImpl::TriWayDotProductI8Impl = &TriWayDotProductI8Simple;
#endif
}
}
diff --git a/library/cpp/dot_product/dot_product.h b/library/cpp/dot_product/dot_product.h
index 154ffaf38f6..cc13ae5d26b 100644
--- a/library/cpp/dot_product/dot_product.h
+++ b/library/cpp/dot_product/dot_product.h
@@ -15,10 +15,17 @@ namespace NDotProductImpl {
extern ui32 (*DotProductUi8Impl)(const ui8* lhs, const ui8* rhs, size_t length) noexcept;
extern i64 (*DotProductI32Impl)(const i32* lhs, const i32* rhs, size_t length) noexcept;
extern float (*DotProductFloatImpl)(const float* lhs, const float* rhs, size_t length) noexcept;
+ extern float (*DotProductFloatI8Impl)(const float* lhs, const i8* rhs, size_t length) noexcept;
extern double (*DotProductDoubleImpl)(const double* lhs, const double* rhs, size_t length) noexcept;
extern TTriWayDotProduct<float> (*TriWayDotProductImpl)
(const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept;
+
+ extern TTriWayDotProductFloatI8 (*TriWayDotProductFloatI8Impl)
+ (const float* lhs, const i8* rhs, size_t length) noexcept;
+
+ extern TTriWayDotProduct<i32> (*TriWayDotProductI8Impl)
+ (const i8* lhs, const i8* rhs, size_t length) noexcept;
}
Y_PURE_FUNCTION
@@ -42,6 +49,11 @@ inline float DotProduct(const float* lhs, const float* rhs, size_t length) noexc
}
Y_PURE_FUNCTION
+inline float DotProduct(const float* lhs, const i8* rhs, size_t length) noexcept {
+ return NDotProductImpl::DotProductFloatI8Impl(lhs, rhs, length);
+}
+
+Y_PURE_FUNCTION
inline double DotProduct(const double* lhs, const double* rhs, size_t length) noexcept {
return NDotProductImpl::DotProductDoubleImpl(lhs, rhs, length);
}
@@ -71,6 +83,24 @@ static inline TTriWayDotProduct<float> TriWayDotProduct(
return TriWayDotProduct(lhs, rhs, length, static_cast<unsigned>(mask));
}
+Y_PURE_FUNCTION
+inline TTriWayDotProductFloatI8 TriWayDotProduct(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return NDotProductImpl::TriWayDotProductFloatI8Impl(lhs, rhs, length);
+}
+
+Y_PURE_FUNCTION
+inline TTriWayDotProduct<i32> TriWayDotProduct(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return NDotProductImpl::TriWayDotProductI8Impl(lhs, rhs, length);
+}
+
namespace NDotProduct {
// Simpler wrapper allowing to use this functions as template argument.
template <typename T>
diff --git a/library/cpp/dot_product/dot_product_avx2.cpp b/library/cpp/dot_product/dot_product_avx2.cpp
index c95888ecda0..0d2c103feea 100644
--- a/library/cpp/dot_product/dot_product_avx2.cpp
+++ b/library/cpp/dot_product/dot_product_avx2.cpp
@@ -155,6 +155,71 @@ namespace {
return HsumI32(sum);
}
+
+ template <class TInput, class TExtend>
+ TTriWayDotProduct<i32> TriWayDotProductInt8Avx2(
+ const TInput* lhs,
+ const TInput* rhs,
+ size_t length,
+ TExtend extend) noexcept
+ {
+ __m256i sumLL = _mm256_setzero_si256();
+ __m256i sumLR = _mm256_setzero_si256();
+ __m256i sumRR = _mm256_setzero_si256();
+
+ if (const auto leftover = length % 16; leftover != 0) {
+ const auto a = _mm_blendv_epi8(
+ Load128i(lhs), _mm_setzero_ps(), BlendMask8[leftover]);
+ const auto b = _mm_blendv_epi8(
+ Load128i(rhs), _mm_setzero_ps(), BlendMask8[leftover]);
+ const auto l = extend(a);
+ const auto r = extend(b);
+ sumLL = _mm256_add_epi32(sumLL, _mm256_madd_epi16(l, l));
+ sumLR = _mm256_add_epi32(sumLR, _mm256_madd_epi16(l, r));
+ sumRR = _mm256_add_epi32(sumRR, _mm256_madd_epi16(r, r));
+ lhs += leftover;
+ rhs += leftover;
+ length -= leftover;
+ }
+
+ while (length >= 32) {
+ const auto l0 = extend(Load128i(lhs));
+ const auto r0 = extend(Load128i(rhs));
+ const auto l1 = extend(Load128i(lhs + 16));
+ const auto r1 = extend(Load128i(rhs + 16));
+ sumLL = _mm256_add_epi32(sumLL, _mm256_add_epi32(_mm256_madd_epi16(l0, l0), _mm256_madd_epi16(l1, l1)));
+ sumLR = _mm256_add_epi32(sumLR, _mm256_add_epi32(_mm256_madd_epi16(l0, r0), _mm256_madd_epi16(l1, r1)));
+ sumRR = _mm256_add_epi32(sumRR, _mm256_add_epi32(_mm256_madd_epi16(r0, r0), _mm256_madd_epi16(r1, r1)));
+ lhs += 32;
+ rhs += 32;
+ length -= 32;
+ }
+
+ if (length >= 16) {
+ const auto l = extend(Load128i(lhs));
+ const auto r = extend(Load128i(rhs));
+ sumLL = _mm256_add_epi32(sumLL, _mm256_madd_epi16(l, l));
+ sumLR = _mm256_add_epi32(sumLR, _mm256_madd_epi16(l, r));
+ sumRR = _mm256_add_epi32(sumRR, _mm256_madd_epi16(r, r));
+ lhs += 16;
+ rhs += 16;
+ length -= 16;
+ }
+
+ TTriWayDotProduct<i32> result{0, 0, 0};
+ result.LL = HsumI32(sumLL);
+ result.LR = HsumI32(sumLR);
+ result.RR = HsumI32(sumRR);
+
+ for (size_t i = 0; i < length; ++i) {
+ const i32 l = lhs[i];
+ const i32 r = rhs[i];
+ result.LL += l * l;
+ result.LR += l * r;
+ result.RR += r * r;
+ }
+ return result;
+ }
}
i32 DotProductAvx2(const i8* lhs, const i8* rhs, size_t length) noexcept {
@@ -403,6 +468,127 @@ TTriWayDotProduct<float> TriWayDotProductAvx2(
}
}
+Y_FORCE_INLINE __m256 LoadFloatI8Rhs8Avx2(const i8* rhs) {
+ const __m128i rhsBytes = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rhs));
+ const __m128i rhsI16 = _mm_cvtepi8_epi16(rhsBytes);
+ const __m128 rhsLo = _mm256_castps256_ps128(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(rhsI16)));
+ const __m128 rhsHi = _mm256_castps256_ps128(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_mm_srli_si128(rhsI16, 8))));
+ return _mm256_set_m128(rhsHi, rhsLo);
+}
+
+Y_FORCE_INLINE void TriWayDotProductFloatI8IterationAvx2(
+ __m256& sumLL,
+ __m256& sumLR,
+ __m256& sumRR,
+ const float* lhs,
+ const i8* rhs) noexcept
+{
+ const __m256 l = _mm256_loadu_ps(lhs);
+ const __m256 r = LoadFloatI8Rhs8Avx2(rhs);
+ sumLL = _mm256_fmadd_ps(l, l, sumLL);
+ sumLR = _mm256_fmadd_ps(l, r, sumLR);
+ sumRR = _mm256_fmadd_ps(r, r, sumRR);
+}
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ __m256 sumLL1 = _mm256_setzero_ps();
+ __m256 sumLR1 = _mm256_setzero_ps();
+ __m256 sumRR1 = _mm256_setzero_ps();
+ __m256 sumLL2 = _mm256_setzero_ps();
+ __m256 sumLR2 = _mm256_setzero_ps();
+ __m256 sumRR2 = _mm256_setzero_ps();
+
+ while (length >= 16) {
+ TriWayDotProductFloatI8IterationAvx2(sumLL1, sumLR1, sumRR1, lhs, rhs);
+ TriWayDotProductFloatI8IterationAvx2(sumLL2, sumLR2, sumRR2, lhs + 8, rhs + 8);
+ lhs += 16;
+ rhs += 16;
+ length -= 16;
+ }
+
+ if (length >= 8) {
+ TriWayDotProductFloatI8IterationAvx2(sumLL1, sumLR1, sumRR1, lhs, rhs);
+ lhs += 8;
+ rhs += 8;
+ length -= 8;
+ }
+
+ sumLL1 = _mm256_add_ps(sumLL1, sumLL2);
+ sumLR1 = _mm256_add_ps(sumLR1, sumLR2);
+ sumRR1 = _mm256_add_ps(sumRR1, sumRR2);
+
+ const auto hsumPs = [](__m256 v) {
+ v = _mm256_add_ps(v, _mm256_permute2f128_ps(v, v, 1));
+ v = _mm256_hadd_ps(v, v);
+ return _mm256_cvtss_f32(_mm256_hadd_ps(v, v));
+ };
+
+ TTriWayDotProductFloatI8 result;
+ result.LL = hsumPs(sumLL1);
+ result.LR = hsumPs(sumLR1);
+ result.RR = hsumPs(sumRR1);
+
+ for (size_t i = 0; i < length; ++i) {
+ const float l = lhs[i];
+ const float r = rhs[i];
+ result.LL += l * l;
+ result.LR += l * r;
+ result.RR += r * r;
+ }
+ return result;
+}
+
+float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept {
+ if (length < 8) {
+ return DotProductSse(lhs, rhs, length);
+ }
+
+ __m256 sum1 = _mm256_setzero_ps();
+ __m256 sum2 = _mm256_setzero_ps();
+
+ while (length >= 16) {
+ sum1 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), LoadFloatI8Rhs8Avx2(rhs), sum1);
+ sum2 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs + 8), LoadFloatI8Rhs8Avx2(rhs + 8), sum2);
+ lhs += 16;
+ rhs += 16;
+ length -= 16;
+ }
+
+ if (length >= 8) {
+ sum1 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), LoadFloatI8Rhs8Avx2(rhs), sum1);
+ lhs += 8;
+ rhs += 8;
+ length -= 8;
+ }
+
+ __m256 sum = _mm256_add_ps(sum1, sum2);
+ sum = _mm256_add_ps(sum, _mm256_permute2f128_ps(sum, sum, 1));
+ sum = _mm256_hadd_ps(sum, sum);
+ float result = _mm256_cvtss_f32(_mm256_hadd_ps(sum, sum));
+
+ for (size_t i = 0; i < length; ++i) {
+ result += lhs[i] * rhs[i];
+ }
+ return result;
+}
+
+TTriWayDotProduct<i32> TriWayDotProductI8Avx2(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ if (length < 16) {
+ return TriWayDotProductI8Sse(lhs, rhs, length);
+ }
+ return TriWayDotProductInt8Avx2(lhs, rhs, length, [](const __m128i x) {
+ return _mm256_cvtepi8_epi16(x);
+ });
+}
+
#elif defined(ARCADIA_SSE)
i32 DotProductAvx2(const i8* lhs, const i8* rhs, size_t length) noexcept {
@@ -434,6 +620,26 @@ TTriWayDotProduct<float> TriWayDotProductAvx2(
return TriWayDotProductSse(lhs, rhs, length, computeRR);
}
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return TriWayDotProductFloatI8Sse(lhs, rhs, length);
+}
+
+float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept {
+ return DotProductSse(lhs, rhs, length);
+}
+
+TTriWayDotProduct<i32> TriWayDotProductI8Avx2(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return TriWayDotProductI8Sse(lhs, rhs, length);
+}
+
#else
i32 DotProductAvx2(const i8* lhs, const i8* rhs, size_t length) noexcept {
@@ -465,4 +671,24 @@ TTriWayDotProduct<float> TriWayDotProductAvx2(
return TriWayDotProductSimple(lhs, rhs, length, computeRR);
}
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return TriWayDotProductFloatI8Simple(lhs, rhs, length);
+}
+
+float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept {
+ return DotProductSimple(lhs, rhs, length);
+}
+
+TTriWayDotProduct<i32> TriWayDotProductI8Avx2(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return TriWayDotProductI8Simple(lhs, rhs, length);
+}
+
#endif
diff --git a/library/cpp/dot_product/dot_product_avx2.h b/library/cpp/dot_product/dot_product_avx2.h
index 68af7e6c8c2..d99dc5a1709 100644
--- a/library/cpp/dot_product/dot_product_avx2.h
+++ b/library/cpp/dot_product/dot_product_avx2.h
@@ -18,8 +18,21 @@ Y_PURE_FUNCTION
float DotProductAvx2(const float* lhs, const float* rhs, size_t length) noexcept;
Y_PURE_FUNCTION
+float DotProductFloatI8Avx2(const float* lhs, const i8* rhs, size_t length) noexcept;
+
+Y_PURE_FUNCTION
double DotProductAvx2(const double* lhs, const double* rhs, size_t length) noexcept;
Y_PURE_FUNCTION
TTriWayDotProduct<float> TriWayDotProductAvx2
(const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept;
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Avx2(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept;
+
+TTriWayDotProduct<i32> TriWayDotProductI8Avx2(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept;
diff --git a/library/cpp/dot_product/dot_product_simple.cpp b/library/cpp/dot_product/dot_product_simple.cpp
index 50afcfe62a3..454dcb3ab0e 100644
--- a/library/cpp/dot_product/dot_product_simple.cpp
+++ b/library/cpp/dot_product/dot_product_simple.cpp
@@ -30,6 +30,14 @@ float DotProductSimple(const float* lhs, const float* rhs, size_t length) noexce
return DotProductSimpleImpl<float, float>(lhs, rhs, length);
}
+float DotProductSimple(const float* lhs, const i8* rhs, size_t length) noexcept {
+ float sum = 0;
+ for (size_t i = 0; i < length; ++i) {
+ sum += lhs[i] * rhs[i];
+ }
+ return sum;
+}
+
double DotProductSimple(const double* lhs, const double* rhs, size_t length) noexcept {
return DotProductSimpleImpl<double, double>(lhs, rhs, length);
}
@@ -74,3 +82,35 @@ TTriWayDotProduct<float> TriWayDotProductSimple(
}
return result;
}
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Simple(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ TTriWayDotProductFloatI8 result;
+ for (size_t i = 0; i < length; ++i) {
+ const float l = lhs[i];
+ const float r = rhs[i];
+ result.LL += l * l;
+ result.LR += l * r;
+ result.RR += r * r;
+ }
+ return result;
+}
+
+TTriWayDotProduct<i32> TriWayDotProductI8Simple(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ TTriWayDotProduct<i32> result{0, 0, 0};
+ for (size_t i = 0; i < length; ++i) {
+ const i32 l = lhs[i];
+ const i32 r = rhs[i];
+ result.LL += l * l;
+ result.LR += l * r;
+ result.RR += r * r;
+ }
+ return result;
+}
diff --git a/library/cpp/dot_product/dot_product_simple.h b/library/cpp/dot_product/dot_product_simple.h
index f951d05f5f0..d0eac9a24b0 100644
--- a/library/cpp/dot_product/dot_product_simple.h
+++ b/library/cpp/dot_product/dot_product_simple.h
@@ -35,6 +35,9 @@ Y_PURE_FUNCTION
float DotProductSimple(const float* lhs, const float* rhs, size_t length) noexcept;
Y_PURE_FUNCTION
+float DotProductSimple(const float* lhs, const i8* rhs, size_t length) noexcept;
+
+Y_PURE_FUNCTION
double DotProductSimple(const double* lhs, const double* rhs, size_t length) noexcept;
Y_PURE_FUNCTION
@@ -46,3 +49,13 @@ TTriWayDotProduct<float> TriWayDotProductSimple(
const float* rhs,
size_t length,
bool computeRR) noexcept;
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Simple(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept;
+
+TTriWayDotProduct<i32> TriWayDotProductI8Simple(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept;
diff --git a/library/cpp/dot_product/dot_product_sse.cpp b/library/cpp/dot_product/dot_product_sse.cpp
index 86f44734876..1cdea0a23c9 100644
--- a/library/cpp/dot_product/dot_product_sse.cpp
+++ b/library/cpp/dot_product/dot_product_sse.cpp
@@ -1,4 +1,5 @@
#include "dot_product_sse.h"
+#include "dot_product_simple.h"
#include <library/cpp/sse/sse.h>
#include <util/system/platform.h>
@@ -173,6 +174,42 @@ float DotProductSse(const float* lhs, const float* rhs, size_t length) noexcept
return res[0] + res[1] + res[2] + res[3];
}
+float DotProductSse(const float* lhs, const i8* rhs, size_t length) noexcept {
+#ifdef _sse4_1_
+ float result = 0;
+ __m128 sum0 = _mm_setzero_ps();
+ __m128 sum1 = _mm_setzero_ps();
+
+ while (length >= 8) {
+ const __m128 lhs0 = _mm_loadu_ps(lhs);
+ const __m128 lhs1 = _mm_loadu_ps(lhs + 4);
+
+ const __m128i rhsBytes = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rhs));
+ const __m128i rhsI16 = _mm_cvtepi8_epi16(rhsBytes);
+ const __m128 rhs0 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(rhsI16));
+ const __m128 rhs1 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_srli_si128(rhsI16, 8)));
+
+ sum0 = _mm_add_ps(sum0, _mm_mul_ps(lhs0, rhs0));
+ sum1 = _mm_add_ps(sum1, _mm_mul_ps(lhs1, rhs1));
+
+ lhs += 8;
+ rhs += 8;
+ length -= 8;
+ }
+
+ alignas(16) float partial[4];
+ _mm_store_ps(partial, _mm_add_ps(sum0, sum1));
+ result = partial[0] + partial[1] + partial[2] + partial[3];
+
+ for (size_t i = 0; i < length; ++i) {
+ result += lhs[i] * rhs[i];
+ }
+ return result;
+#else
+ return DotProductSimple(lhs, rhs, length);
+#endif
+}
+
double DotProductSse(const double* lhs, const double* rhs, size_t length) noexcept {
__m128d sum1 = _mm_setzero_pd();
__m128d sum2 = _mm_setzero_pd();
@@ -315,4 +352,130 @@ TTriWayDotProduct<float> TriWayDotProductSse(
}
}
+#ifdef _sse4_1_
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Sse(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ __m128 sumLL0 = _mm_setzero_ps();
+ __m128 sumLR0 = _mm_setzero_ps();
+ __m128 sumRR0 = _mm_setzero_ps();
+ __m128 sumLL1 = _mm_setzero_ps();
+ __m128 sumLR1 = _mm_setzero_ps();
+ __m128 sumRR1 = _mm_setzero_ps();
+
+ while (length >= 8) {
+ const __m128 lhs0 = _mm_loadu_ps(lhs);
+ const __m128 lhs1 = _mm_loadu_ps(lhs + 4);
+
+ const __m128i rhsBytes = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(rhs));
+ const __m128i rhsI16 = _mm_cvtepi8_epi16(rhsBytes);
+ const __m128 rhs0 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(rhsI16));
+ const __m128 rhs1 = _mm_cvtepi32_ps(_mm_cvtepi16_epi32(_mm_srli_si128(rhsI16, 8)));
+
+ sumLL0 = _mm_add_ps(sumLL0, _mm_mul_ps(lhs0, lhs0));
+ sumLR0 = _mm_add_ps(sumLR0, _mm_mul_ps(lhs0, rhs0));
+ sumRR0 = _mm_add_ps(sumRR0, _mm_mul_ps(rhs0, rhs0));
+
+ sumLL1 = _mm_add_ps(sumLL1, _mm_mul_ps(lhs1, lhs1));
+ sumLR1 = _mm_add_ps(sumLR1, _mm_mul_ps(lhs1, rhs1));
+ sumRR1 = _mm_add_ps(sumRR1, _mm_mul_ps(rhs1, rhs1));
+
+ lhs += 8;
+ rhs += 8;
+ length -= 8;
+ }
+
+ sumLL0 = _mm_add_ps(sumLL0, sumLL1);
+ sumLR0 = _mm_add_ps(sumLR0, sumLR1);
+ sumRR0 = _mm_add_ps(sumRR0, sumRR1);
+
+ alignas(16) float partial[4];
+ _mm_store_ps(partial, sumLL0);
+ TTriWayDotProductFloatI8 result;
+ result.LL = partial[0] + partial[1] + partial[2] + partial[3];
+ _mm_store_ps(partial, sumLR0);
+ result.LR = partial[0] + partial[1] + partial[2] + partial[3];
+ _mm_store_ps(partial, sumRR0);
+ result.RR = partial[0] + partial[1] + partial[2] + partial[3];
+
+ for (size_t i = 0; i < length; ++i) {
+ const float l = lhs[i];
+ const float r = rhs[i];
+ result.LL += l * l;
+ result.LR += l * r;
+ result.RR += r * r;
+ }
+ return result;
+}
+
+TTriWayDotProduct<i32> TriWayDotProductI8Sse(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ const __m128i zero = _mm_setzero_si128();
+ __m128i sumLL = zero;
+ __m128i sumLR = zero;
+ __m128i sumRR = zero;
+
+ while (length >= 16) {
+ const __m128i lVec = _mm_loadu_si128(reinterpret_cast<const __m128i*>(lhs));
+ const __m128i rVec = _mm_loadu_si128(reinterpret_cast<const __m128i*>(rhs));
+
+ const __m128i lLo = _mm_cvtepi8_epi16(lVec);
+ const __m128i rLo = _mm_cvtepi8_epi16(rVec);
+ const __m128i lHi = _mm_cvtepi8_epi16(_mm_alignr_epi8(lVec, lVec, 8));
+ const __m128i rHi = _mm_cvtepi8_epi16(_mm_alignr_epi8(rVec, rVec, 8));
+
+ sumLL = _mm_add_epi32(sumLL, _mm_add_epi32(_mm_madd_epi16(lLo, lLo), _mm_madd_epi16(lHi, lHi)));
+ sumLR = _mm_add_epi32(sumLR, _mm_add_epi32(_mm_madd_epi16(lLo, rLo), _mm_madd_epi16(lHi, rHi)));
+ sumRR = _mm_add_epi32(sumRR, _mm_add_epi32(_mm_madd_epi16(rLo, rLo), _mm_madd_epi16(rHi, rHi)));
+
+ lhs += 16;
+ rhs += 16;
+ length -= 16;
+ }
+
+ alignas(16) i32 partial[4];
+ _mm_store_si128(reinterpret_cast<__m128i*>(partial), sumLL);
+ TTriWayDotProduct<i32> result;
+ result.LL = partial[0] + partial[1] + partial[2] + partial[3];
+ _mm_store_si128(reinterpret_cast<__m128i*>(partial), sumLR);
+ result.LR = partial[0] + partial[1] + partial[2] + partial[3];
+ _mm_store_si128(reinterpret_cast<__m128i*>(partial), sumRR);
+ result.RR = partial[0] + partial[1] + partial[2] + partial[3];
+
+ for (size_t i = 0; i < length; ++i) {
+ const i32 l = lhs[i];
+ const i32 r = rhs[i];
+ result.LL += l * l;
+ result.LR += l * r;
+ result.RR += r * r;
+ }
+ return result;
+}
+
+#else
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Sse(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return TriWayDotProductFloatI8Simple(lhs, rhs, length);
+}
+
+TTriWayDotProduct<i32> TriWayDotProductI8Sse(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept
+{
+ return TriWayDotProductI8Simple(lhs, rhs, length);
+}
+
+#endif
+
#endif // ARCADIA_SSE
diff --git a/library/cpp/dot_product/dot_product_sse.h b/library/cpp/dot_product/dot_product_sse.h
index 8bc26ef75b0..9dbe0da3fd5 100644
--- a/library/cpp/dot_product/dot_product_sse.h
+++ b/library/cpp/dot_product/dot_product_sse.h
@@ -18,8 +18,21 @@ Y_PURE_FUNCTION
float DotProductSse(const float* lhs, const float* rhs, size_t length) noexcept;
Y_PURE_FUNCTION
+float DotProductSse(const float* lhs, const i8* rhs, size_t length) noexcept;
+
+Y_PURE_FUNCTION
double DotProductSse(const double* lhs, const double* rhs, size_t length) noexcept;
Y_PURE_FUNCTION
TTriWayDotProduct<float> TriWayDotProductSse
(const float* lhs, const float* rhs, size_t length, bool computeRR) noexcept;
+
+TTriWayDotProductFloatI8 TriWayDotProductFloatI8Sse(
+ const float* lhs,
+ const i8* rhs,
+ size_t length) noexcept;
+
+TTriWayDotProduct<i32> TriWayDotProductI8Sse(
+ const i8* lhs,
+ const i8* rhs,
+ size_t length) noexcept;
diff --git a/library/cpp/dot_product/dot_product_ut.cpp b/library/cpp/dot_product/dot_product_ut.cpp
index caf74a832d0..6d07d1a3a31 100644
--- a/library/cpp/dot_product/dot_product_ut.cpp
+++ b/library/cpp/dot_product/dot_product_ut.cpp
@@ -1,11 +1,13 @@
#include "dot_product.h"
#include "dot_product_simple.h"
+#include "dot_product_avx2.h"
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/sse/sse.h>
#include <util/generic/vector.h>
#include <util/random/fast.h>
+#include <util/system/cpu_id.h>
#include <cmath>
@@ -37,6 +39,15 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) {
return sum;
}
+ template <class Res, class TLhs, class TRhs>
+ Res SimpleDotProductMixed(const TLhs* lhs, const TRhs* rhs, size_t length) {
+ Res sum = 0;
+ for (size_t i = 0; i < length; ++i) {
+ sum += static_cast<Res>(lhs[i]) * static_cast<Res>(rhs[i]);
+ }
+ return sum;
+ }
+
Y_UNIT_TEST(TestDotProduct8) {
TVector<i8> a(100);
FillWithRandomNumbers(a.data(), 179, 100);
@@ -93,6 +104,72 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) {
}
}
+ Y_UNIT_TEST(TestDotProductFloatI8) {
+ TVector<float> floats(100);
+ FillWithRandomFloats(floats.data(), 179, 100);
+ TVector<i8> bytes(100);
+ FillWithRandomNumbers(bytes.data(), 239, 100);
+
+ constexpr double MIXED_EPSILON = 1e-4;
+ for (size_t i = 0; i < 30; ++i) {
+ for (size_t length = 1; length + i + 1 < floats.size(); ++length) {
+ const float expected = SimpleDotProductMixed<float, float, i8>(floats.data() + i, bytes.data() + i, length);
+ UNIT_ASSERT(std::fabs(DotProduct(floats.data() + i, bytes.data() + i, length) - expected) < MIXED_EPSILON);
+ UNIT_ASSERT(std::fabs(DotProductSimple(floats.data() + i, bytes.data() + i, length) - expected) < EPSILON);
+ if (NX86::HaveAVX2() && NX86::HaveFMA()) {
+ UNIT_ASSERT(std::fabs(DotProductFloatI8Avx2(floats.data() + i, bytes.data() + i, length) - expected) < MIXED_EPSILON);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(TestTriWayDotProductFloatI8) {
+ TVector<float> floats(100);
+ FillWithRandomFloats(floats.data(), 179, 100);
+ TVector<i8> bytes(100);
+ FillWithRandomNumbers(bytes.data(), 239, 100);
+
+ constexpr double MIXED_EPSILON = 1e-4;
+ for (size_t i = 0; i < 30; ++i) {
+ for (size_t length = 1; length + i + 1 < floats.size(); ++length) {
+ const auto expected = TriWayDotProductFloatI8Simple(floats.data() + i, bytes.data() + i, length);
+ const auto actual = TriWayDotProduct(floats.data() + i, bytes.data() + i, length);
+ UNIT_ASSERT_DOUBLES_EQUAL(expected.LL, actual.LL, MIXED_EPSILON);
+ UNIT_ASSERT_DOUBLES_EQUAL(expected.LR, actual.LR, MIXED_EPSILON);
+ UNIT_ASSERT_DOUBLES_EQUAL(expected.RR, actual.RR, MIXED_EPSILON);
+ if (NX86::HaveAVX2() && NX86::HaveFMA()) {
+ const auto avx2 = TriWayDotProductFloatI8Avx2(floats.data() + i, bytes.data() + i, length);
+ UNIT_ASSERT_DOUBLES_EQUAL(expected.LL, avx2.LL, MIXED_EPSILON);
+ UNIT_ASSERT_DOUBLES_EQUAL(expected.LR, avx2.LR, MIXED_EPSILON);
+ UNIT_ASSERT_DOUBLES_EQUAL(expected.RR, avx2.RR, MIXED_EPSILON);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(TestTriWayDotProductI8) {
+ TVector<i8> a(100);
+ FillWithRandomNumbers(a.data(), 179, 100);
+ TVector<i8> b(100);
+ FillWithRandomNumbers(b.data(), 239, 100);
+
+ for (size_t i = 0; i < 30; ++i) {
+ for (size_t length = 1; length + i + 1 < a.size(); ++length) {
+ const auto expected = TriWayDotProductI8Simple(a.data() + i, b.data() + i, length);
+ const auto actual = TriWayDotProduct(a.data() + i, b.data() + i, length);
+ UNIT_ASSERT_EQUAL(expected.LL, actual.LL);
+ UNIT_ASSERT_EQUAL(expected.LR, actual.LR);
+ UNIT_ASSERT_EQUAL(expected.RR, actual.RR);
+ if (NX86::HaveAVX2() && NX86::HaveFMA()) {
+ const auto avx2 = TriWayDotProductI8Avx2(a.data() + i, b.data() + i, length);
+ UNIT_ASSERT_EQUAL(expected.LL, avx2.LL);
+ UNIT_ASSERT_EQUAL(expected.LR, avx2.LR);
+ UNIT_ASSERT_EQUAL(expected.RR, avx2.RR);
+ }
+ }
+ }
+ }
+
Y_UNIT_TEST(TestL2NormSqaredf) {
TVector<float> a(100);
FillWithRandomFloats(a.data(), 179, 100);
@@ -172,12 +249,14 @@ Y_UNIT_TEST_SUITE(TDocProductTestSuite) {
UNIT_ASSERT_EQUAL(DotProduct(static_cast<const i8*>(nullptr), nullptr, 0), 0);
UNIT_ASSERT_EQUAL(DotProduct(static_cast<const ui8*>(nullptr), nullptr, 0), 0);
UNIT_ASSERT_EQUAL(DotProduct(static_cast<const i32*>(nullptr), nullptr, 0), 0);
- UNIT_ASSERT(std::abs(DotProduct(static_cast<const float*>(nullptr), nullptr, 0)) < EPSILON);
+ UNIT_ASSERT(std::abs(DotProduct(static_cast<const float*>(nullptr), static_cast<const float*>(nullptr), 0)) < EPSILON);
+ UNIT_ASSERT(std::abs(DotProduct(static_cast<const float*>(nullptr), static_cast<const i8*>(nullptr), 0)) < EPSILON);
UNIT_ASSERT(std::abs(DotProduct(static_cast<const double*>(nullptr), nullptr, 0)) < EPSILON);
UNIT_ASSERT_EQUAL(DotProductSimple(static_cast<const i8*>(nullptr), nullptr, 0), 0);
UNIT_ASSERT_EQUAL(DotProductSimple(static_cast<const ui8*>(nullptr), nullptr, 0), 0);
UNIT_ASSERT_EQUAL(DotProductSimple(static_cast<const i32*>(nullptr), nullptr, 0), 0);
- UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const float*>(nullptr), nullptr, 0)) < EPSILON);
+ UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const float*>(nullptr), static_cast<const float*>(nullptr), 0)) < EPSILON);
+ UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const float*>(nullptr), static_cast<const i8*>(nullptr), 0)) < EPSILON);
UNIT_ASSERT(std::abs(DotProductSimple(static_cast<const double*>(nullptr), nullptr, 0)) < EPSILON);
}
diff --git a/library/cpp/http/server/conn.cpp b/library/cpp/http/server/conn.cpp
index 38a76c4c309..4b72c3f77f0 100644
--- a/library/cpp/http/server/conn.cpp
+++ b/library/cpp/http/server/conn.cpp
@@ -1,47 +1,38 @@
#include "conn.h"
#include <util/network/socket.h>
-#include <util/stream/buffered.h>
-class THttpServerConn::TImpl {
-public:
- inline TImpl(const TSocket& s, size_t outputBufferSize)
- : S_(s)
- , SI_(S_)
- , SO_(S_)
- , BO_(&SO_, outputBufferSize)
- , HI_(&SI_)
- , HO_(&BO_, &HI_)
- {
- }
-
- inline ~TImpl() {
- }
-
- inline THttpInput* Input() noexcept {
- return &HI_;
- }
+namespace {
+ class TBlockingSocketStreams: public THttpServerConn::ISocketStreams {
+ public:
+ explicit TBlockingSocketStreams(const TSocket& s)
+ : Socket_(s)
+ , Input_(Socket_)
+ , Output_(Socket_)
+ {
+ }
- inline THttpOutput* Output() noexcept {
- return &HO_;
- }
+ IInputStream* Input() override {
+ return &Input_;
+ }
- inline void Reset() {
- if (S_ != INVALID_SOCKET) {
- // send RST packet to client
- S_.SetLinger(true, 0);
- S_.Close();
+ IOutputStream* Output() override {
+ return &Output_;
}
- }
-private:
- TSocket S_;
- TSocketInput SI_;
- TSocketOutput SO_;
- TBufferedOutput BO_;
- THttpInput HI_;
- THttpOutput HO_;
-};
+ void Reset() override {
+ if (Socket_ != INVALID_SOCKET) {
+ // send RST packet to client
+ Socket_.SetLinger(true, 0);
+ Socket_.Close();
+ }
+ }
+ private:
+ TSocket Socket_;
+ TSocketInput Input_;
+ TSocketOutput Output_;
+ };
+}
THttpServerConn::THttpServerConn(const TSocket& s)
: THttpServerConn(s, s.MaximumTransferUnit())
@@ -49,21 +40,21 @@ THttpServerConn::THttpServerConn(const TSocket& s)
}
THttpServerConn::THttpServerConn(const TSocket& s, size_t outputBufferSize)
- : Impl_(new TImpl(s, outputBufferSize))
+ : THttpServerConn(MakeHolder<TBlockingSocketStreams>(s), outputBufferSize)
{
}
-THttpServerConn::~THttpServerConn() {
-}
-
-THttpInput* THttpServerConn::Input() noexcept {
- return Impl_->Input();
+THttpServerConn::THttpServerConn(THolder<ISocketStreams> socketStreams, size_t outputBufferSize)
+ : SocketStreams_(std::move(socketStreams))
+ , BufferedOutput_(SocketStreams_->Output(), outputBufferSize)
+ , HttpInput_(SocketStreams_->Input())
+ , HttpOutput_(&BufferedOutput_, &HttpInput_)
+{
}
-THttpOutput* THttpServerConn::Output() noexcept {
- return Impl_->Output();
+THttpServerConn::~THttpServerConn() {
}
void THttpServerConn::Reset() {
- return Impl_->Reset();
+ return SocketStreams_->Reset();
}
diff --git a/library/cpp/http/server/conn.h b/library/cpp/http/server/conn.h
index 3aa5329af42..62a3e86ac95 100644
--- a/library/cpp/http/server/conn.h
+++ b/library/cpp/http/server/conn.h
@@ -2,18 +2,37 @@
#include <library/cpp/http/io/stream.h>
#include <util/generic/ptr.h>
+#include <util/stream/buffered.h>
+class IInputStream;
+class IOutputStream;
class TSocket;
/// Потоки ввода/вывода для получения запросов и отправки ответов HTTP-сервера.
class THttpServerConn {
public:
- explicit THttpServerConn(const TSocket& s);
+ class ISocketStreams {
+ public:
+ virtual ~ISocketStreams() {}
+
+ virtual IInputStream* Input() = 0;
+ virtual IOutputStream* Output() = 0;
+ virtual void Reset() = 0;
+ };
+public:
+ THttpServerConn(const TSocket& s);
THttpServerConn(const TSocket& s, size_t outputBufferSize);
+ THttpServerConn(THolder<ISocketStreams> socketStreams, size_t outputBufferSize);
+
~THttpServerConn();
- THttpInput* Input() noexcept;
- THttpOutput* Output() noexcept;
+ THttpInput* Input() noexcept {
+ return &HttpInput_;
+ }
+
+ THttpOutput* Output() noexcept {
+ return &HttpOutput_;
+ }
inline const THttpInput* Input() const noexcept {
return const_cast<THttpServerConn*>(this)->Input();
@@ -30,8 +49,10 @@ public:
}
void Reset();
-
private:
- class TImpl;
- THolder<TImpl> Impl_;
+ THolder<ISocketStreams> SocketStreams_;
+
+ TBufferedOutput BufferedOutput_;
+ THttpInput HttpInput_;
+ THttpOutput HttpOutput_;
};
diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp
index 13088f5e442..79a1194c770 100644
--- a/library/cpp/http/server/http.cpp
+++ b/library/cpp/http/server/http.cpp
@@ -725,6 +725,10 @@ void TClientRequest::ResetConnection() {
}
}
+THolder<THttpServerConn> TClientRequest::CreateHttpConnection(const TSocket& s, size_t outputBufferSize) {
+ return MakeHolder<THttpServerConn>(s, outputBufferSize);
+}
+
void TClientRequest::Process(void* ThreadSpecificResource) {
THolder<TClientRequest> this_(this);
@@ -733,11 +737,7 @@ void TClientRequest::Process(void* ThreadSpecificResource) {
try {
if (!HttpConn_) {
const size_t outputBufferSize = HttpServ()->Options().OutputBufferSize;
- if (outputBufferSize) {
- HttpConn_.Reset(new THttpServerConn(Socket(), outputBufferSize));
- } else {
- HttpConn_.Reset(new THttpServerConn(Socket()));
- }
+ HttpConn_ = CreateHttpConnection(Socket(), outputBufferSize ? outputBufferSize : Socket().MaximumTransferUnit());
auto maxRequestsPerConnection = HttpServ()->Options().MaxRequestsPerConnection;
HttpConn_->Output()->EnableKeepAlive(HttpServ()->Options().KeepAliveEnabled && (!maxRequestsPerConnection || Conn_->ReceivedRequests < maxRequestsPerConnection));
diff --git a/library/cpp/http/server/http.h b/library/cpp/http/server/http.h
index 62dfd5a50fa..5b23cb21a00 100644
--- a/library/cpp/http/server/http.h
+++ b/library/cpp/http/server/http.h
@@ -141,6 +141,8 @@ private:
Y_UNUSED(ThreadSpecificResource);
return true;
}
+ virtual THolder<THttpServerConn> CreateHttpConnection(const TSocket& s, size_t outputBuffer);
+
void Process(void* ThreadSpecificResource) override;
public:
diff --git a/library/cpp/http/server/http_ut.cpp b/library/cpp/http/server/http_ut.cpp
index 0fc44fa5983..b3280745929 100644
--- a/library/cpp/http/server/http_ut.cpp
+++ b/library/cpp/http/server/http_ut.cpp
@@ -7,6 +7,7 @@
#include <util/generic/cast.h>
#include <util/stream/output.h>
#include <util/stream/zlib.h>
+#include <util/stream/tee.h>
#include <util/system/datetime.h>
#include <util/system/mutex.h>
#include <util/random/random.h>
@@ -1016,4 +1017,112 @@ Y_UNIT_TEST_SUITE(THttpServerTest) {
t2->Join();
UNIT_ASSERT_EQUAL_C(results, (THashSet<TString>({"Zoooo", "TTL Exceed"})), "Results is {" + ToString(results) + "}");
}
+
+ Y_UNIT_TEST(TestCustomSocketStreams) {
+ // Check that TClientRequest::CreateHttpConnection override works
+
+ class TCustomSocketStreamsServer: public THttpServer::ICallBack {
+ public:
+ TStringStream InputCopy;
+ TStringStream OutputCopy;
+ private:
+ class TTeeInput
+ : public IInputStream
+ {
+ public:
+ TTeeInput(IInputStream* s, IOutputStream* copy)
+ : S_(s)
+ , Copy_(copy)
+ {
+ }
+
+ size_t DoRead(void* buf, size_t len) override {
+ void* begin = buf;
+ size_t res = S_->Read(buf, len);
+ Copy_->Write(begin, res);
+ return res;
+ }
+
+ private:
+ IInputStream* const S_;
+ IOutputStream* const Copy_;
+ };
+
+ class TBlockingSocketStreams: public THttpServerConn::ISocketStreams {
+ public:
+ explicit TBlockingSocketStreams(const TSocket& s, TCustomSocketStreamsServer* server)
+ : Socket_(s)
+ , Input_(Socket_)
+ , TI_(&Input_, &server->InputCopy)
+ , Output_(Socket_)
+ , TO_(&Output_, &server->OutputCopy)
+ {
+ }
+
+ IInputStream* Input() override {
+ return &TI_;
+ }
+
+ IOutputStream* Output() override {
+ return &TO_;
+ }
+
+ void Reset() override {}
+ private:
+ TSocket Socket_;
+ TSocketInput Input_;
+ TTeeInput TI_;
+ TSocketOutput Output_;
+ TTeeOutput TO_;
+ };
+
+ class TRequest: public TClientRequest {
+ public:
+ TRequest(TCustomSocketStreamsServer* server)
+ : Server_(server)
+ {
+ }
+
+ bool Reply(void* /*tsr*/) override {
+ Output() << "HTTP/1.1 200 Ok\r\n\r\n";
+ Output().Finish();
+ return true;
+ }
+
+ THolder<THttpServerConn> CreateHttpConnection(const TSocket& s, size_t outputBuffer) override {
+ return MakeHolder<THttpServerConn>(MakeHolder<TBlockingSocketStreams>(s, Server_), outputBuffer);
+ }
+ private:
+ TCustomSocketStreamsServer* const Server_;
+ };
+
+ public:
+ TClientRequest* CreateClient() override {
+ return new TRequest(this);
+ }
+
+ void OnException() override {
+ ExceptionMessage = CurrentExceptionMessage();
+ }
+
+ TString ExceptionMessage;
+ };
+
+ TPortManager portManager;
+ const ui16 port = portManager.GetPort();
+ TCustomSocketStreamsServer server;
+ THttpServer::TOptions options(port);
+ options.nThreads = 1;
+ options.MaxConnections = 2;
+ THttpServer srv(&server, options);
+
+ UNIT_ASSERT(srv.Start());
+
+ TSocket socket(TNetworkAddress("localhost", port), TDuration::Seconds(10));
+
+ SendRequest(socket, port);
+
+ UNIT_ASSERT_STRINGS_EQUAL(server.InputCopy.Str(), TStringBuilder() << "GET / HTTP/1.1\r\nHost: localhost:" << port << "\r\nConnection: Keep-Alive\r\n\r\n");
+ UNIT_ASSERT_STRINGS_EQUAL(server.OutputCopy.Str(), TStringBuilder() << "HTTP/1.1 200 Ok\r\nConnection: Keep-Alive\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n");
+ }
}
diff --git a/library/cpp/protobuf/json/proto/ya.make b/library/cpp/protobuf/json/proto/ya.make
index b565c78d160..fd9e0563bc8 100644
--- a/library/cpp/protobuf/json/proto/ya.make
+++ b/library/cpp/protobuf/json/proto/ya.make
@@ -5,4 +5,6 @@ SRCS(
enum_options.proto
)
+INCLUDE_TAGS(DOCS_PROTO)
+
END()
diff --git a/library/cpp/streams/lzma/lzma_ut.cpp b/library/cpp/streams/lzma/lzma_ut.cpp
index 847e98d1caa..dbcd7d78fac 100644
--- a/library/cpp/streams/lzma/lzma_ut.cpp
+++ b/library/cpp/streams/lzma/lzma_ut.cpp
@@ -10,7 +10,7 @@ class TStrokaByOneByte: public IZeroCopyInput {
public:
TStrokaByOneByte(const TString& s)
: Data(s)
- , Pos(s.data())
+ , Pos(Data.data())
{
}
diff --git a/library/cpp/streams/lzma/ut/ya.make b/library/cpp/streams/lzma/ut/ya.make
index 99b7e5e401a..88fdb91dd1b 100644
--- a/library/cpp/streams/lzma/ut/ya.make
+++ b/library/cpp/streams/lzma/ut/ya.make
@@ -1,5 +1,7 @@
UNITTEST_FOR(library/cpp/streams/lzma)
+NO_COW()
+
SRCS(
lzma_ut.cpp
)
diff --git a/library/cpp/yt/memory/ref_counted-inl.h b/library/cpp/yt/memory/ref_counted-inl.h
index 3d5b212bf96..94acfc6669b 100644
--- a/library/cpp/yt/memory/ref_counted-inl.h
+++ b/library/cpp/yt/memory/ref_counted-inl.h
@@ -8,12 +8,48 @@
#include <util/system/sanitizers.h>
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+#include <util/system/types.h>
+#endif
+
#include <stdlib.h>
namespace NYT {
////////////////////////////////////////////////////////////////////////////////
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+
+namespace NDetail {
+
+// A distinctive marker held by a live signature (after address salting).
+constexpr ui64 RefCountedAliveSignatureMagic = 0xa11e0b1ec70a11e0ULL;
+
+// A distinct poison written into the signature once the object is destroyed.
+constexpr ui64 RefCountedDeadSignatureMagic = 0xdeadbeefdeadbeefULL;
+
+//! The alive signature for a signature word living at #address: the magic XOR-ed
+//! with the word's own address.
+//!
+//! The signature is embedded as the first word of every TRefCounter: the ctor
+//! stamps ComputeRefCountedAliveSignature(&signature) and strong-death overwrites
+//! it with RefCountedDeadSignatureMagic. A coredump walker tells a live object
+//! from freed-but-unreclaimed memory: for a word at address S,
+//! *S == ComputeRefCountedAliveSignature(S) means alive,
+//! *S == RefCountedDeadSignatureMagic means freed. The address salt keeps a
+//! signature from validating if copied verbatim elsewhere (it carries the old
+//! address) and makes a chance match of arbitrary memory a ~2^-64 event.
+Y_FORCE_INLINE ui64 ComputeRefCountedAliveSignature(const void* address) noexcept
+{
+ return RefCountedAliveSignatureMagic ^ reinterpret_cast<uintptr_t>(address);
+}
+
+} // namespace NDetail
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
// TODO(babenko): move to hazard pointers
void RetireHazardPointer(
void* protectedPtr,
@@ -190,6 +226,19 @@ Y_FORCE_INLINE int TRefCounter::GetRefCount() const noexcept
return StrongCount_.load(std::memory_order::acquire);
}
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+
+Y_FORCE_INLINE TRefCounter::TRefCounter() noexcept
+ : Signature_(NDetail::ComputeRefCountedAliveSignature(this))
+{ }
+
+Y_FORCE_INLINE ui64 TRefCounter::GetSignature() const noexcept
+{
+ return Signature_;
+}
+
+#endif
+
Y_FORCE_INLINE void TRefCounter::Ref(int n) const noexcept
{
YT_ASSERT(n >= 0);
@@ -238,6 +287,11 @@ Y_FORCE_INLINE bool TRefCounter::Unref(int n) const
if (oldStrongCount == n) {
std::atomic_thread_fence(std::memory_order::acquire);
NSan::Acquire(&StrongCount_);
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+ // Last strong ref gone: the object is about to be destroyed. Poison now so
+ // a core walker sees freed-but-unreclaimed memory as dead.
+ Signature_ = NDetail::RefCountedDeadSignatureMagic;
+#endif
return true;
} else {
return false;
diff --git a/library/cpp/yt/memory/ref_counted.h b/library/cpp/yt/memory/ref_counted.h
index 154bbd14659..e82d22936e4 100644
--- a/library/cpp/yt/memory/ref_counted.h
+++ b/library/cpp/yt/memory/ref_counted.h
@@ -39,6 +39,10 @@ private:
class TRefCounter
{
public:
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+ TRefCounter() noexcept;
+#endif
+
//! Returns current number of strong references to the object.
/*!
* Note that you should never ever use this method in production code.
@@ -69,7 +73,22 @@ public:
//! Decrements the weak reference counter.
bool WeakUnref() const;
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+ //! Returns the raw signature word. Used only by unit tests; the coredump
+ //! walker reads the word directly from memory.
+ ui64 GetSignature() const noexcept;
+#endif
+
private:
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+ // An address-salted liveness marker. NB: Must be the first member -- the
+ // salt is its own address and a core validator keys off its offset. Stamped
+ // alive by the ctor (see -inl); poisoned at strong-death (see Unref).
+ // TRefCounter is already non-copyable (atomics below), so the salted value
+ // can't be copied astray.
+ mutable ui64 Signature_;
+#endif
+
// NB: Must be 64 bit as TAtomicIntrusivePtr grabs refs in 64K batches.
using TRefCount = i64;
mutable std::atomic<TRefCount> StrongCount_ = 1;
diff --git a/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp b/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp
index 2b665b41473..86b6341f915 100644
--- a/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp
+++ b/library/cpp/yt/memory/unittests/intrusive_ptr_ut.cpp
@@ -691,5 +691,43 @@ TEST(TIntrusivePtrTest, HeterogeneousLookup)
////////////////////////////////////////////////////////////////////////////////
+#ifdef YT_ENABLE_REF_COUNTED_SIGNATURE
+
+TEST(TRefCountedSignatureTest, FinalType)
+{
+ // Final, non-derived type: New lays a TRefCounter right before the object.
+ struct TObject final
+ {
+ ui64 Data = 0;
+ };
+
+ auto obj = New<TObject>();
+ const auto* refCounter = GetRefCounter(obj.Get());
+
+ // The signature is salted purely by the counter's own address.
+ EXPECT_EQ(refCounter->GetSignature(), NDetail::ComputeRefCountedAliveSignature(refCounter));
+ EXPECT_NE(refCounter->GetSignature(), NDetail::RefCountedDeadSignatureMagic);
+}
+
+TEST(TRefCountedSignatureTest, DerivedType)
+{
+ // Polymorphic (TRefCounted-derived) type: the counter is a base subobject.
+ struct TObject
+ : public TRefCounted
+ {
+ ui64 Data = 0;
+ };
+
+ auto obj = New<TObject>();
+ const auto* refCounter = GetRefCounter(obj.Get());
+
+ EXPECT_EQ(refCounter->GetSignature(), NDetail::ComputeRefCountedAliveSignature(refCounter));
+ EXPECT_NE(refCounter->GetSignature(), NDetail::RefCountedDeadSignatureMagic);
+}
+
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+
} // namespace
} // namespace NYT
diff --git a/library/cpp/yt/misc/port.h b/library/cpp/yt/misc/port.h
index e54a2450ce9..b7b350f4581 100644
--- a/library/cpp/yt/misc/port.h
+++ b/library/cpp/yt/misc/port.h
@@ -8,6 +8,13 @@
#define YT_ENABLE_REF_COUNTED_TRACKING
#endif
+#if !defined(NDEBUG) && !defined(YT_DISABLE_REF_COUNTED_SIGNATURE)
+ // This define embeds a salted, poisonable signature into every TRefCounter
+ // (see ref_counted.h). It lets a coredump walker tell a live ref-counted object
+ // from freed-but-unreclaimed memory.
+ #define YT_ENABLE_REF_COUNTED_SIGNATURE
+#endif
+
// This define enables logging with TRACE level. You can still disable trace logging
// for particular TU by discarding this macro identifier.
#define YT_ENABLE_TRACE_LOGGING
diff --git a/library/cpp/yt/string/string.cpp b/library/cpp/yt/string/string.cpp
index 1da50b4ccdd..e4a2afeca5e 100644
--- a/library/cpp/yt/string/string.cpp
+++ b/library/cpp/yt/string/string.cpp
@@ -65,6 +65,26 @@ TString CamelCaseToUnderscoreCase(TStringBuf str)
////////////////////////////////////////////////////////////////////////////////
+std::string AsciiStringToLower(TStringBuf value)
+{
+ std::string result(value.size(), '\0');
+ for (size_t index = 0; index < value.size(); ++index) {
+ result[index] = ::AsciiToLower(value[index]);
+ }
+ return result;
+}
+
+std::string AsciiStringToUpper(TStringBuf value)
+{
+ std::string result(value.size(), '\0');
+ for (size_t index = 0; index < value.size(); ++index) {
+ result[index] = ::AsciiToUpper(value[index]);
+ }
+ return result;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
[[nodiscard]] TStringBuf TrimLeadingWhitespaces(TStringBuf str Y_LIFETIME_BOUND)
{
auto begin = str.find_first_not_of(' ');
@@ -292,7 +312,7 @@ size_t TCaseInsensitiveStringHasher::operator()(TStringBuf arg) const
{
auto compute = [&] (char* buffer) {
for (size_t index = 0; index < arg.length(); ++index) {
- buffer[index] = AsciiToLower(arg[index]);
+ buffer[index] = ::AsciiToLower(arg[index]);
}
return ComputeHash(TStringBuf(buffer, arg.length()));
};
diff --git a/library/cpp/yt/string/string.h b/library/cpp/yt/string/string.h
index 346ec04ef4d..968ac3a0554 100644
--- a/library/cpp/yt/string/string.h
+++ b/library/cpp/yt/string/string.h
@@ -148,6 +148,15 @@ TString UnderscoreCaseToCamelCase(TStringBuf str);
void CamelCaseToUnderscoreCase(TStringBuilderBase* builder, TStringBuf str);
TString CamelCaseToUnderscoreCase(TStringBuf str);
+//! ASCII case conversion returning |std::string|.
+//! TODO(babenko): likely a temporary workaround; these exist only because util's
+//! |to_lower|/|to_upper| are |TString|-based (forcing a |TString| round-trip for
+//! |std::string| callers). Drop once util gains |std::string| support.
+//! Returns a copy of |value| with every ASCII letter lowercased; non-ASCII bytes are left intact.
+std::string AsciiStringToLower(TStringBuf value);
+//! Returns a copy of |value| with every ASCII letter uppercased; non-ASCII bytes are left intact.
+std::string AsciiStringToUpper(TStringBuf value);
+
[[nodiscard]] TStringBuf TrimLeadingWhitespaces(TStringBuf str Y_LIFETIME_BOUND);
[[nodiscard]] TStringBuf Trim(TStringBuf str Y_LIFETIME_BOUND, TStringBuf whitespaces = " ");
diff --git a/library/cpp/yt/threading/rw_spin_lock.cpp b/library/cpp/yt/threading/rw_spin_lock.cpp
index efc0e90167a..3150c1aa132 100644
--- a/library/cpp/yt/threading/rw_spin_lock.cpp
+++ b/library/cpp/yt/threading/rw_spin_lock.cpp
@@ -41,7 +41,8 @@ void TUncheckedReaderWriterSpinLock::AcquireWriterSlow() noexcept
namespace {
-// NB: Reading this after destruction is UB, but we can't deal with Static Initialization Order Fiasco in any other feasible way :(
+// NB: Reading this after destruction is UB, but we can't deal with
+// Static Initialization Order Fiasco in any other feasible way :(
YT_DEFINE_THREAD_LOCAL(bool, ThreadLockTrackerDestroyed, false);
class TThreadLockTracker
@@ -61,13 +62,17 @@ public:
~TThreadLockTracker()
{
ThreadLockTrackerDestroyed() = true;
- };
+ }
private:
THashSet<TCheckedReaderWriterSpinLock*> ThreadLocksAcquired_;
};
-YT_DEFINE_THREAD_LOCAL(TThreadLockTracker, ThreadLockTracker);
+YT_PREVENT_TLS_CACHING static TThreadLockTracker& ThreadLockTracker()
+{
+ static thread_local TThreadLockTracker Instance;
+ return Instance;
+}
} // namespace