aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoralexvru <alexvru@ydb.tech>2023-01-22 20:56:44 +0300
committeralexvru <alexvru@ydb.tech>2023-01-22 20:56:44 +0300
commit8d667b6cf729f3b77e7ac3ee36524f30afd484f8 (patch)
treefa3b30b16be0bef3e692cca5dd3e55bae2b4596c
parentff6e75892efa3822427135ec727e2bf07b4fb89e (diff)
downloadydb-8d667b6cf729f3b77e7ac3ee36524f30afd484f8.tar.gz
Introduce LoadedKeys interval set for correct key processing
-rw-r--r--ydb/core/blob_depot/CMakeLists.darwin.txt1
-rw-r--r--ydb/core/blob_depot/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/core/blob_depot/CMakeLists.linux.txt1
-rw-r--r--ydb/core/blob_depot/closed_interval_set.h166
-rw-r--r--ydb/core/blob_depot/closed_interval_set_ut.cpp106
-rw-r--r--ydb/core/blob_depot/data.cpp39
-rw-r--r--ydb/core/blob_depot/data.h70
-rw-r--r--ydb/core/blob_depot/data_load.cpp79
-rw-r--r--ydb/core/blob_depot/data_mon.cpp1
-rw-r--r--ydb/core/blob_depot/data_resolve.cpp69
-rw-r--r--ydb/core/blob_depot/garbage_collection.cpp2
-rw-r--r--ydb/core/blob_depot/given_id_range_ut.cpp177
-rw-r--r--ydb/core/blob_depot/op_commit_blob_seq.cpp9
-rw-r--r--ydb/core/blob_depot/ut/CMakeLists.darwin.txt66
-rw-r--r--ydb/core/blob_depot/ut/CMakeLists.linux-aarch64.txt69
-rw-r--r--ydb/core/blob_depot/ut/CMakeLists.linux.txt71
-rw-r--r--ydb/core/blob_depot/ut/CMakeLists.txt15
17 files changed, 841 insertions, 101 deletions
diff --git a/ydb/core/blob_depot/CMakeLists.darwin.txt b/ydb/core/blob_depot/CMakeLists.darwin.txt
index 47ebcb53fd..b7013389cf 100644
--- a/ydb/core/blob_depot/CMakeLists.darwin.txt
+++ b/ydb/core/blob_depot/CMakeLists.darwin.txt
@@ -7,6 +7,7 @@
add_subdirectory(agent)
+add_subdirectory(ut)
add_library(ydb-core-blob_depot)
target_link_libraries(ydb-core-blob_depot PUBLIC
diff --git a/ydb/core/blob_depot/CMakeLists.linux-aarch64.txt b/ydb/core/blob_depot/CMakeLists.linux-aarch64.txt
index 5d5087cec1..04727d74a4 100644
--- a/ydb/core/blob_depot/CMakeLists.linux-aarch64.txt
+++ b/ydb/core/blob_depot/CMakeLists.linux-aarch64.txt
@@ -7,6 +7,7 @@
add_subdirectory(agent)
+add_subdirectory(ut)
add_library(ydb-core-blob_depot)
target_link_libraries(ydb-core-blob_depot PUBLIC
diff --git a/ydb/core/blob_depot/CMakeLists.linux.txt b/ydb/core/blob_depot/CMakeLists.linux.txt
index 5d5087cec1..04727d74a4 100644
--- a/ydb/core/blob_depot/CMakeLists.linux.txt
+++ b/ydb/core/blob_depot/CMakeLists.linux.txt
@@ -7,6 +7,7 @@
add_subdirectory(agent)
+add_subdirectory(ut)
add_library(ydb-core-blob_depot)
target_link_libraries(ydb-core-blob_depot PUBLIC
diff --git a/ydb/core/blob_depot/closed_interval_set.h b/ydb/core/blob_depot/closed_interval_set.h
new file mode 100644
index 0000000000..df28dfaadb
--- /dev/null
+++ b/ydb/core/blob_depot/closed_interval_set.h
@@ -0,0 +1,166 @@
+#pragma once
+
+#include "defs.h"
+
+namespace NKikimr {
+
+ template<typename T>
+ class TClosedIntervalSet {
+ struct TByLeft {
+ const T& Value;
+ };
+
+ struct TByRight {
+ const T& Value;
+ };
+
+ struct TInterval {
+ T Left;
+ T Right;
+
+ TInterval(T&& left, T&& right)
+ : Left(std::move(left))
+ , Right(std::move(right))
+ {}
+
+ struct TCompare {
+ using is_transparent = void;
+
+ bool operator ()(const TInterval& x, const TInterval& y) const { return x.Left < y.Left; }
+ bool operator ()(const TByLeft& x, const TInterval& y) const { return x.Value < y.Left; }
+ bool operator ()(const TInterval& x, const TByLeft& y) const { return x.Left < y.Value; }
+ bool operator ()(const TByRight& x, const TInterval& y) const { return x.Value < y.Right; }
+ bool operator ()(const TInterval& x, const TByRight& y) const { return x.Right < y.Value; }
+ };
+ };
+ std::set<TInterval, typename TInterval::TCompare> Intervals;
+
+ public:
+ TClosedIntervalSet() = default;
+ TClosedIntervalSet(const TClosedIntervalSet&) = default;
+ TClosedIntervalSet(TClosedIntervalSet&&) = default;
+
+ TClosedIntervalSet& operator =(const TClosedIntervalSet&) = default;
+ TClosedIntervalSet& operator =(TClosedIntervalSet&&) = default;
+
+ TClosedIntervalSet& operator |=(const std::pair<T, T>& range) { AddRange(range); return *this; }
+ TClosedIntervalSet& operator |=(std::pair<T, T>&& range) { AddRange(std::move(range)); return *this; }
+
+ template<typename TRange>
+ void AddRange(TRange&& range) {
+ auto&& [left, right] = range;
+ const auto leftIt = Intervals.lower_bound(TByRight{left});
+ const auto rightIt = Intervals.upper_bound(TByLeft{right});
+ if (leftIt == rightIt) {
+ Intervals.emplace(std::move(left), std::move(right));
+ } else {
+ auto& current = const_cast<TInterval&>(*leftIt);
+ auto& last = const_cast<TInterval&>(*std::prev(rightIt));
+ if (left < current.Left) {
+ current.Left = std::move(left);
+ }
+ if (current.Right < right || current.Right < last.Right) {
+ current.Right = right < last.Right ? std::move(last.Right) : std::move(right);
+ }
+ Intervals.erase(std::next(leftIt), rightIt);
+ }
+ }
+
+ TClosedIntervalSet& operator -=(const TClosedIntervalSet& other) {
+ auto myIt = Intervals.begin();
+ auto otherIt = other.Intervals.begin();
+ while (myIt != Intervals.end() && otherIt != other.Intervals.end()) {
+ auto& my = const_cast<TInterval&>(*myIt);
+
+ if (my.Right < otherIt->Left) {
+ ++myIt;
+ if (myIt != Intervals.end() && myIt->Right < otherIt->Left) {
+ myIt = Intervals.lower_bound(TByRight{otherIt->Left});
+ }
+ } else if (otherIt->Right < my.Left) {
+ ++otherIt;
+ if (otherIt != other.Intervals.end() && otherIt->Right < my.Left) {
+ otherIt = other.Intervals.lower_bound(TByRight{my.Left});
+ }
+ } else if (otherIt->Left <= my.Left) {
+ if (my.Right <= otherIt->Right) {
+ myIt = Intervals.erase(myIt);
+ } else {
+ my.Left = otherIt->Right;
+ ++otherIt;
+ }
+ } else if (my.Right <= otherIt->Right) {
+ my.Right = otherIt->Left;
+ ++myIt;
+ } else {
+ if (otherIt->Left < otherIt->Right) {
+ myIt = Intervals.emplace_hint(std::next(myIt), T(otherIt->Right), std::exchange(my.Right, otherIt->Left));
+ }
+ ++otherIt;
+ }
+ }
+ return *this;
+ }
+
+ // returns the first subrange of the full subtraction result
+ std::optional<std::pair<T, T>> PartialSubtract(const TClosedIntervalSet& other) const {
+ if (auto myIt = Intervals.begin(); myIt != Intervals.end()) {
+ const T *myLeft = &myIt->Left;
+ const T *myRight = &myIt->Right;
+
+ for (auto otherIt = other.Intervals.begin(); otherIt != other.Intervals.end(); ) {
+ if (*myRight < otherIt->Left) {
+ return std::make_pair(*myLeft, *myRight);
+ } else if (otherIt->Right < *myLeft) {
+ ++otherIt;
+ if (otherIt != other.Intervals.end() && otherIt->Right < *myLeft) {
+ otherIt = other.Intervals.lower_bound(TByRight{*myLeft});
+ }
+ } else if (otherIt->Left <= *myLeft) {
+ if (*myRight <= otherIt->Right) {
+ ++myIt;
+ if (myIt == Intervals.end()) {
+ return std::nullopt;
+ }
+ std::tie(myLeft, myRight) = std::make_pair(&myIt->Left, &myIt->Right);
+ } else {
+ myLeft = &otherIt->Right;
+ ++otherIt;
+ }
+ } else if (*myRight <= otherIt->Right) {
+ return std::make_pair(*myLeft, otherIt->Left);
+ } else {
+ if (otherIt->Left < otherIt->Right) {
+ return std::make_pair(*myLeft, otherIt->Left);
+ }
+ ++otherIt;
+ }
+ }
+
+ return std::make_pair(*myLeft, *myRight);
+ } else {
+ return std::nullopt;
+ }
+ }
+
+ operator bool() const {
+ return !Intervals.empty();
+ }
+
+ bool operator [](const T& pt) const {
+ const auto it = Intervals.lower_bound(TByRight{pt});
+ return it != Intervals.end() && it->Left <= pt;
+ }
+
+ template<typename TCallback>
+ bool operator ()(TCallback&& callback) const {
+ for (const auto& i : Intervals) {
+ if (!callback(i.Left, i.Right)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ };
+
+} // NKikimr
diff --git a/ydb/core/blob_depot/closed_interval_set_ut.cpp b/ydb/core/blob_depot/closed_interval_set_ut.cpp
new file mode 100644
index 0000000000..cb6d70a555
--- /dev/null
+++ b/ydb/core/blob_depot/closed_interval_set_ut.cpp
@@ -0,0 +1,106 @@
+#include "closed_interval_set.h"
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NKikimr;
+
+using T = TClosedIntervalSet<ui8>;
+
+TString ToString(const T& ivs) {
+ TStringStream s("[");
+ bool flag = true;
+ ivs([&](ui8 first, ui8 last) {
+ s << (std::exchange(flag, false) ? "" : " ") << (unsigned)first << "-" << (unsigned)last;
+ return true;
+ });
+ s << "]";
+ return s.Str();
+}
+
+ui64 Convert(const T& ivs) {
+ ui64 res = 0;
+ ivs([&](ui8 first, ui8 last) {
+ const ui64 mask = ((ui64(1) << (last - first + 1)) - 1) << first;
+ UNIT_ASSERT_VALUES_EQUAL_C(res & mask, 0, ToString(ivs));
+ res |= mask;
+ return true;
+ });
+ return res;
+}
+
+T Make(ui64 mask) {
+ const ui64 original = mask;
+ unsigned pos = 0;
+ T res;
+ while (mask) {
+ const ui64 bit = mask & 1;
+ const unsigned n = CountTrailingZeroBits(mask ^ -bit);
+ if (bit) {
+ res |= {pos, pos + n - 1};
+ }
+ mask >>= n;
+ pos += n;
+ }
+ UNIT_ASSERT_VALUES_EQUAL(Convert(res), original);
+ return res;
+}
+
+Y_UNIT_TEST_SUITE(ClosedIntervalSet) {
+
+ Y_UNIT_TEST(Union) {
+ for (ui32 i = 0; i < 4096; ++i) {
+ for (ui32 begin = 0; begin <= 12; ++begin) {
+ for (ui32 end = begin; end <= 12; ++end) {
+ T x = Make(i);
+ x |= {begin, end};
+ UNIT_ASSERT_VALUES_EQUAL(Convert(x), i | ((1 << (end - begin + 1)) - 1) << begin);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(Difference) {
+ for (ui32 i = 0; i < 1024; ++i) {
+ for (ui32 j = 0; j < 1024; ++j) {
+ T x = Make(i);
+ T y = Make(j);
+ ui64 expected = i & ~j;
+
+ ui64 xLeft = 0, xRight = 0;
+ x([&](ui8 left, ui8 right) {
+ xLeft |= ui64(1) << left;
+ xRight |= ui64(1) << right;
+ return true;
+ });
+
+ y([&](ui8 left, ui8 right) {
+ expected |= i & ~xLeft & ui64(1) << left;
+ expected |= i & ~xRight & ui64(1) << right;
+ return true;
+ });
+
+ x -= y;
+ UNIT_ASSERT_VALUES_EQUAL(Convert(x), expected);
+
+ std::optional<std::pair<ui8, ui8>> firstRange;
+ x([&](ui8 first, ui8 last) {
+ UNIT_ASSERT(!firstRange);
+ firstRange = {first, last};
+ return false;
+ });
+
+ const auto interval = Make(i).PartialSubtract(y);
+ UNIT_ASSERT_EQUAL(interval, firstRange);
+ }
+ }
+ }
+
+ Y_UNIT_TEST(Contains) {
+ for (ui32 i = 0; i < 4096; ++i) {
+ T x = Make(i);
+ for (ui32 j = 0; j < 12; ++j) {
+ UNIT_ASSERT_VALUES_EQUAL(x[j], (i >> j) & 1);
+ }
+ }
+ }
+
+}
diff --git a/ydb/core/blob_depot/data.cpp b/ydb/core/blob_depot/data.cpp
index fd1c25f69f..5c34c16cdf 100644
--- a/ydb/core/blob_depot/data.cpp
+++ b/ydb/core/blob_depot/data.cpp
@@ -202,7 +202,7 @@ namespace NKikimr::NBlobDepot {
void TData::UpdateKey(const TKey& key, const NKikimrBlobDepot::TEvCommitBlobSeq::TItem& item,
NTabletFlatExecutor::TTransactionContext& txc, void *cookie) {
STLOG(PRI_DEBUG, BLOB_DEPOT, BDT10, "UpdateKey", (Id, Self->GetLogId()), (Key, key), (Item, item));
- Y_VERIFY(Loaded || IsKeyLoaded(key));
+ Y_VERIFY(IsKeyLoaded(key));
UpdateKey(key, txc, cookie, "UpdateKey", [&](TValue& value, bool inserted) {
if (!inserted) { // update value items
value.Meta = item.GetMeta();
@@ -225,7 +225,7 @@ namespace NKikimr::NBlobDepot {
void TData::BindToBlob(const TKey& key, TBlobSeqId blobSeqId, NTabletFlatExecutor::TTransactionContext& txc, void *cookie) {
STLOG(PRI_DEBUG, BLOB_DEPOT, BDT49, "BindToBlob", (Id, Self->GetLogId()), (Key, key), (BlobSeqId, blobSeqId));
- Y_VERIFY(Loaded || IsKeyLoaded(key));
+ Y_VERIFY(IsKeyLoaded(key));
UpdateKey(key, txc, cookie, "BindToBlob", [&](TValue& value, bool inserted) {
if (inserted || value.GoingToAssimilate) {
auto *chain = value.ValueChain.Add();
@@ -308,33 +308,14 @@ namespace NKikimr::NBlobDepot {
return it->second;
}
- void TData::AddLoadSkip(TKey key) {
- Y_VERIFY(!Loaded);
- if (!LastLoadedKey || *LastLoadedKey < key) {
- LoadSkip.insert(std::move(key));
- }
- }
-
- void TData::AddDataOnLoad(TKey key, TString value, bool uncertainWrite, bool skip) {
- if (skip) {
- Y_VERIFY_DEBUG(!LastLoadedKey || *LastLoadedKey < key);
- AddLoadSkip(key);
- } else {
- // delete keys that might have been loaded and deleted while we were still loading data
- LoadSkip.erase(LoadSkip.begin(), LoadSkip.lower_bound(key));
-
- // check if we have to skip currently loaded key
- if (LoadSkip.erase(key)) {
- return;
- }
- }
+ void TData::AddDataOnLoad(TKey key, TString value, bool uncertainWrite) {
+ Y_VERIFY_S(!IsKeyLoaded(key), "Id# " << Self->GetLogId() << " Key# " << key.ToString());
NKikimrBlobDepot::TValue proto;
const bool success = proto.ParseFromString(value);
Y_VERIFY(success);
- STLOG(PRI_DEBUG, BLOB_DEPOT, BDT79, "AddDataOnLoad", (Id, Self->GetLogId()), (Key, key), (Value, proto),
- (Skip, skip));
+ STLOG(PRI_DEBUG, BLOB_DEPOT, BDT79, "AddDataOnLoad", (Id, Self->GetLogId()), (Key, key), (Value, proto));
// we can only add key that is not loaded before; if key exists, it MUST have been loaded from the dataset
const auto [it, inserted] = Data.try_emplace(std::move(key), std::move(proto), uncertainWrite);
@@ -352,9 +333,8 @@ namespace NKikimr::NBlobDepot {
}
bool TData::AddDataOnDecommit(const TEvBlobStorage::TEvAssimilateResult::TBlob& blob,
- NTabletFlatExecutor::TTransactionContext& txc, void *cookie, bool suppressLoadedCheck) {
- Y_VERIFY_S(suppressLoadedCheck || Loaded || IsKeyLoaded(TKey(blob.Id)), "Id# " << Self->GetLogId()
- << " Blob# " << blob.ToString());
+ NTabletFlatExecutor::TTransactionContext& txc, void *cookie) {
+ Y_VERIFY_S(IsKeyLoaded(TKey(blob.Id)), "Id# " << Self->GetLogId() << " Blob# " << blob.ToString());
return UpdateKey(TKey(blob.Id), txc, cookie, "AddDataOnDecommit", [&](TValue& value, bool inserted) {
bool change = inserted;
@@ -394,7 +374,7 @@ namespace NKikimr::NBlobDepot {
}
bool TData::UpdateKeepState(TKey key, EKeepState keepState, NTabletFlatExecutor::TTransactionContext& txc, void *cookie) {
- Y_VERIFY(Loaded || IsKeyLoaded(key));
+ Y_VERIFY(IsKeyLoaded(key));
return UpdateKey(std::move(key), txc, cookie, "UpdateKeepState", [&](TValue& value, bool inserted) {
STLOG(PRI_DEBUG, BLOB_DEPOT, BDT51, "UpdateKeepState", (Id, Self->GetLogId()), (Key, key),
(KeepState, keepState), (Value, value));
@@ -484,8 +464,7 @@ namespace NKikimr::NBlobDepot {
STLOG(PRI_DEBUG, BLOB_DEPOT, BDT18, "OnBarrierShift", (Id, Self->GetLogId()), (TabletId, tabletId),
(Channel, int(channel)), (Hard, hard), (Previous, previous), (Current, current), (MaxItems, maxItems));
- Y_VERIFY(Loaded || (LastLoadedKey && *LastLoadedKey > TKey(TLogoBlobID(tabletId, current.Generation(), current.Step(),
- channel, TLogoBlobID::MaxBlobSize, TLogoBlobID::MaxCookie, TLogoBlobID::MaxPartId, TLogoBlobID::MaxCrcMode))));
+ Y_VERIFY(Loaded);
const TData::TKey first(TLogoBlobID(tabletId, previous.Generation(), previous.Step(), channel, 0, 0));
const TData::TKey last(TLogoBlobID(tabletId, current.Generation(), current.Step(), channel,
diff --git a/ydb/core/blob_depot/data.h b/ydb/core/blob_depot/data.h
index 33a883c055..254293f244 100644
--- a/ydb/core/blob_depot/data.h
+++ b/ydb/core/blob_depot/data.h
@@ -2,6 +2,7 @@
#include "defs.h"
#include "blob_depot_tablet.h"
+#include "closed_interval_set.h"
#include <util/generic/hash_multi_map.h>
@@ -21,11 +22,18 @@ namespace NKikimr::NBlobDepot {
static constexpr size_t TypeLenByteIdx = 31;
static constexpr size_t MaxInlineStringLen = TypeLenByteIdx;
- static constexpr char BlobIdType = 32;
- static constexpr char StringType = 33;
+ static constexpr ui8 MinType = 32;
+ static constexpr ui8 BlobIdType = 33;
+ static constexpr ui8 StringType = 34;
+ static constexpr ui8 MaxType = 255;
static_assert(sizeof(Data) == 32);
+ private:
+ explicit TKey(ui8 type) {
+ Data.Type = type;
+ }
+
public:
TKey() {
Data.Type = EncodeInlineStringLenAsTypeByte(0);
@@ -74,6 +82,9 @@ namespace NKikimr::NBlobDepot {
Reset();
}
+ static TKey Min() { return TKey(MinType); }
+ static TKey Max() { return TKey(MaxType); }
+
TKey& operator =(const TKey& other) {
if (this != &other) {
if (Data.Type == StringType && other.Data.Type == StringType) {
@@ -121,8 +132,12 @@ namespace NKikimr::NBlobDepot {
TString MakeBinaryKey() const {
if (Data.Type == BlobIdType) {
return GetBlobId().AsBinaryString();
- } else {
+ } else if (Data.Type <= MaxInlineStringLen || Data.Type == StringType) {
return TString(GetStringBuf());
+ } else if (Data.Type == MinType) {
+ return {};
+ } else {
+ Y_FAIL();
}
}
@@ -143,22 +158,43 @@ namespace NKikimr::NBlobDepot {
void Output(IOutputStream& s) const {
if (Data.Type == BlobIdType) {
s << GetBlobId();
+ } else if (Data.Type == MinType) {
+ s << "<min>";
+ } else if (Data.Type == MaxType) {
+ s << "<max>";
} else {
s << EscapeC(GetStringBuf());
}
}
static int Compare(const TKey& x, const TKey& y) {
- if (x.Data.Type == BlobIdType && y.Data.Type == BlobIdType) {
- return x.GetBlobId() < y.GetBlobId() ? -1 : y.GetBlobId() < x.GetBlobId() ? 1 : 0;
- } else if (x.Data.Type == BlobIdType) {
+ const ui8 xType = x.Data.Type <= MaxInlineStringLen ? StringType : x.Data.Type;
+ const ui8 yType = y.Data.Type <= MaxInlineStringLen ? StringType : y.Data.Type;
+ if (xType < yType) {
return -1;
- } else if (y.Data.Type == BlobIdType) {
+ } else if (yType < xType) {
return 1;
} else {
- const TStringBuf sbx = x.GetStringBuf();
- const TStringBuf sby = y.GetStringBuf();
- return sbx < sby ? -1 : sby < sbx ? 1 : 0;
+ switch (xType) {
+ case StringType: {
+ const TStringBuf sbx = x.GetStringBuf();
+ const TStringBuf sby = y.GetStringBuf();
+ return sbx < sby ? -1 : sby < sbx ? 1 : 0;
+ }
+
+ case BlobIdType: {
+ const TLogoBlobID& xId = x.GetBlobId();
+ const TLogoBlobID& yId = y.GetBlobId();
+ return xId < yId ? -1 : yId < xId ? 1 : 0;
+ }
+
+ case MinType:
+ case MaxType:
+ return 0;
+
+ default:
+ Y_FAIL();
+ }
}
}
@@ -193,8 +229,10 @@ namespace NKikimr::NBlobDepot {
TStringBuf GetStringBuf() const {
if (Data.Type == StringType) {
return GetString();
- } else {
+ } else if (Data.Type <= MaxInlineStringLen) {
return TStringBuf(reinterpret_cast<const char*>(Data.Bytes), DecodeInlineStringLenFromTypeByte(Data.Type));
+ } else {
+ Y_FAIL();
}
}
@@ -346,10 +384,9 @@ namespace NKikimr::NBlobDepot {
bool Loaded = false;
std::map<TKey, TValue> Data;
- std::set<TKey> LoadSkip; // keys to skip while loading
+ TClosedIntervalSet<TKey> LoadedKeys; // keys that are already scanned and loaded in the local database
THashMap<TLogoBlobID, ui32> RefCount;
THashMap<std::tuple<ui8, ui32>, TRecordsPerChannelGroup> RecordsPerChannelGroup;
- std::optional<TKey> LastLoadedKey; // keys are being loaded in ascending order
std::optional<TLogoBlobID> LastAssimilatedBlobId;
ui64 TotalStoredDataSize = 0;
ui64 TotalStoredTrashSize = 0;
@@ -454,10 +491,9 @@ namespace NKikimr::NBlobDepot {
TRecordsPerChannelGroup& GetRecordsPerChannelGroup(TLogoBlobID id);
TRecordsPerChannelGroup& GetRecordsPerChannelGroup(ui8 channel, ui32 groupId);
- void AddLoadSkip(TKey key);
- void AddDataOnLoad(TKey key, TString value, bool uncertainWrite, bool skip);
+ void AddDataOnLoad(TKey key, TString value, bool uncertainWrite);
bool AddDataOnDecommit(const TEvBlobStorage::TEvAssimilateResult::TBlob& blob,
- NTabletFlatExecutor::TTransactionContext& txc, void *cookie, bool suppressLoadedCheck = false);
+ NTabletFlatExecutor::TTransactionContext& txc, void *cookie);
void AddTrashOnLoad(TLogoBlobID id);
void AddGenStepOnLoad(ui8 channel, ui32 groupId, TGenStep issuedGenStep, TGenStep confirmedGenStep);
@@ -498,7 +534,7 @@ namespace NKikimr::NBlobDepot {
void StartLoad();
void OnLoadComplete();
bool IsLoaded() const { return Loaded; }
- bool IsKeyLoaded(const TKey& key) const { return key <= LastLoadedKey || Data.contains(key) || LoadSkip.contains(key); }
+ bool IsKeyLoaded(const TKey& key) const { return Loaded || LoadedKeys[key]; }
void Handle(TEvBlobDepot::TEvResolve::TPtr ev);
void Handle(TEvBlobStorage::TEvRangeResult::TPtr ev);
diff --git a/ydb/core/blob_depot/data_load.cpp b/ydb/core/blob_depot/data_load.cpp
index 019cd93a16..010ae46e2c 100644
--- a/ydb/core/blob_depot/data_load.cpp
+++ b/ydb/core/blob_depot/data_load.cpp
@@ -8,7 +8,6 @@ namespace NKikimr::NBlobDepot {
class TData::TTxDataLoad : public NTabletFlatExecutor::TTransactionBase<TBlobDepot> {
std::optional<TString> LastTrashKey;
- std::optional<TString> LastDataKey;
bool TrashLoaded = false;
bool SuccessorTx = true;
@@ -22,7 +21,6 @@ namespace NKikimr::NBlobDepot {
TTxDataLoad(TTxDataLoad& predecessor)
: TTransactionBase(predecessor.Self)
, LastTrashKey(std::move(predecessor.LastTrashKey))
- , LastDataKey(std::move(predecessor.LastDataKey))
, TrashLoaded(predecessor.TrashLoaded)
{}
@@ -67,15 +65,73 @@ namespace NKikimr::NBlobDepot {
TrashLoaded = true;
}
- auto addData = [this](const auto& key, const auto& rows) {
- auto k = TData::TKey::FromBinaryKey(key, Self->Config);
- Self->Data->AddDataOnLoad(k, rows.template GetValue<Schema::Data::Value>(),
- rows.template GetValueOrDefault<Schema::Data::UncertainWrite>(), false);
- Y_VERIFY(!Self->Data->LastLoadedKey || *Self->Data->LastLoadedKey < k);
- Self->Data->LastLoadedKey = std::move(k);
- };
- if (!load(db.Table<Schema::Data>(), LastDataKey, addData)) {
- return progress;
+ for (;;) {
+ // calculate a set of keys we need to load
+ TClosedIntervalSet<TKey> needed;
+ needed |= {TKey::Min(), TKey::Max()};
+ const auto interval = needed.PartialSubtract(Self->Data->LoadedKeys);
+ if (!interval) {
+ break;
+ }
+
+ const auto& [first, last] = *interval;
+
+ auto makeNeeded = [&] {
+ TStringStream s("{");
+ bool flag = true;
+ needed([&](const TKey& first, const TKey& last) {
+ s << (std::exchange(flag, false) ? "" : "-");
+ first.Output(s);
+ last.Output(s << '-');
+ return true;
+ });
+ s << '}';
+ return s.Str();
+ };
+ STLOG(PRI_DEBUG, BLOB_DEPOT, BDT83, "TTxDataLoad iteration", (Id, Self->GetLogId()), (Needed, makeNeeded()),
+ (FirstKey, first), (LastKey, last));
+
+ bool status = true;
+ std::optional<TKey> lastKey; // the actual last processed key
+
+ auto table = db.Table<Schema::Data>().GreaterOrEqual(first.MakeBinaryKey());
+ static constexpr ui64 PrechargeRows = 10'000;
+ static constexpr ui64 PrechargeBytes = 1'000'000;
+ if (!table.Precharge(PrechargeRows, PrechargeBytes)) {
+ status = false;
+ } else if (auto rows = table.Select(); !rows.IsReady()) {
+ status = false;
+ } else {
+ for (;;) {
+ if (!rows.IsValid()) { // finished reading the range
+ lastKey.emplace(last);
+ break;
+ }
+ auto key = TKey::FromBinaryKey(rows.GetKey(), Self->Config);
+ lastKey.emplace(key);
+ if (last <= key) { // stop iteration -- we are getting out of range
+ break;
+ } else if (first < key && !Self->Data->IsKeyLoaded(key)) {
+ Self->Data->AddDataOnLoad(std::move(key), rows.template GetValue<Schema::Data::Value>(),
+ rows.template GetValueOrDefault<Schema::Data::UncertainWrite>());
+ progress = true;
+ }
+ if (!rows.Next()) {
+ status = false;
+ break;
+ }
+ }
+ }
+
+ STLOG(PRI_DEBUG, BLOB_DEPOT, BDT84, "TTxDataLoad iteration complete", (Id, Self->GetLogId()),
+ (FirstKey, first), (LastKey, lastKey), (Status, status), (Progress, progress));
+
+ if (lastKey) {
+ Self->Data->LoadedKeys |= {first, *lastKey};
+ }
+ if (!status) {
+ return progress;
+ }
}
SuccessorTx = false; // everything loaded
@@ -100,7 +156,6 @@ namespace NKikimr::NBlobDepot {
void TData::OnLoadComplete() {
Loaded = true;
- LoadSkip.clear();
Self->OnDataLoadComplete();
for (auto& [key, record] : RecordsPerChannelGroup) {
record.CollectIfPossible(this);
diff --git a/ydb/core/blob_depot/data_mon.cpp b/ydb/core/blob_depot/data_mon.cpp
index e7f68c4eb4..9a286b6781 100644
--- a/ydb/core/blob_depot/data_mon.cpp
+++ b/ydb/core/blob_depot/data_mon.cpp
@@ -15,7 +15,6 @@ namespace NKikimr::NBlobDepot {
DIV_CLASS("panel-body") {
KEYVALUE_TABLE({
KEYVALUE_P("Loaded", Loaded ? "true" : "false");
- KEYVALUE_P("Last loaded key", LastLoadedKey ? LastLoadedKey->ToString() : "<null>");
KEYVALUE_P("Last assimilated blob id", LastAssimilatedBlobId ? LastAssimilatedBlobId->ToString() : "<null>");
KEYVALUE_P("Data size, number of keys", Data.size());
KEYVALUE_P("RefCount size, number of blobs", RefCount.size());
diff --git a/ydb/core/blob_depot/data_resolve.cpp b/ydb/core/blob_depot/data_resolve.cpp
index 1f25d09a43..8ab2a2d3e1 100644
--- a/ydb/core/blob_depot/data_resolve.cpp
+++ b/ydb/core/blob_depot/data_resolve.cpp
@@ -178,7 +178,7 @@ namespace NKikimr::NBlobDepot {
const auto& blob = ResolveDecommitContext.DecommitBlobs.front();
if (Self->Data->LastAssimilatedBlobId < blob.Id) {
- numItemsRemain -= Self->Data->AddDataOnDecommit(blob, txc, this, true);
+ numItemsRemain -= Self->Data->AddDataOnDecommit(blob, txc, this);
}
ResolveDecommitContext.DecommitBlobs.pop_front();
}
@@ -265,42 +265,31 @@ namespace NKikimr::NBlobDepot {
}
}
- if (end && Self->Data->LastLoadedKey && *end <= *Self->Data->LastLoadedKey) {
+ TClosedIntervalSet<TKey> needed;
+ needed |= {begin.value_or(TKey::Min()), end.value_or(TKey::Max())};
+ needed -= Self->Data->LoadedKeys;
+ if (!needed) {
STLOG(PRI_DEBUG, BLOB_DEPOT, BDT76, "TTxResolve: skipping subrange", (Id, Self->GetLogId()),
(Sender, Request->Sender), (Cookie, Request->Cookie), (ItemIndex, ItemIndex));
- continue; // key is already loaded
+ continue;
}
- if (Self->Data->LastLoadedKey && begin <= Self->Data->LastLoadedKey && !(flags & EScanFlags::REVERSE)) {
- Y_VERIFY(!end || *Self->Data->LastLoadedKey < *end);
-
- // special case -- forward scan and we have some data in memory
- auto callback = [&](const TKey& key, const TValue& /*value*/) {
- LastScannedKey = key;
- STLOG(PRI_DEBUG, BLOB_DEPOT, BDT83, "TTxResolve: skipping key in memory", (Id, Self->GetLogId()),
- (Sender, Request->Sender), (Cookie, Request->Cookie), (ItemIndex, ItemIndex), (Key, key));
- return !ResolveDecommitContext.DecommitBlobs.empty() || ++NumKeysRead != maxKeys;
- };
- if (!Self->Data->ScanRange(begin, Self->Data->LastLoadedKey, flags | EScanFlags::INCLUDE_END, callback)) {
- STLOG(PRI_DEBUG, BLOB_DEPOT, BDT84, "TTxResolve: skipping remaining subrange", (Id, Self->GetLogId()),
- (Sender, Request->Sender), (Cookie, Request->Cookie), (ItemIndex, ItemIndex));
- continue; // we have read all the keys required (MaxKeys limit hit)
- }
-
- // adjust range beginning
- begin = Self->Data->LastLoadedKey;
- flags &= ~EScanFlags::INCLUDE_BEGIN;
- }
+ std::optional<TKey> boundary;
auto processRange = [&](auto table) {
+ bool done = false;
for (auto rowset = table.Select();; rowset.Next()) {
if (!rowset.IsReady()) {
- return false;
+ return done;
} else if (!rowset.IsValid()) {
- // no more keys in our direction
+ // no more keys in our direction -- we have scanned full requested range
+ boundary.emplace(flags & EScanFlags::REVERSE
+ ? begin.value_or(TKey::Min())
+ : end.value_or(TKey::Max()));
return true;
}
auto key = TKey::FromBinaryKey(rowset.template GetValue<Schema::Data::Key>(), Self->Config);
+ boundary.emplace(key);
if (key != LastScannedKey) {
LastScannedKey = key;
progress = true;
@@ -313,19 +302,18 @@ namespace NKikimr::NBlobDepot {
if (!isKeyLoaded) {
Self->Data->AddDataOnLoad(key, rowset.template GetValue<Schema::Data::Value>(),
- rowset.template GetValueOrDefault<Schema::Data::UncertainWrite>(), true);
+ rowset.template GetValueOrDefault<Schema::Data::UncertainWrite>());
}
const bool matchBegin = !begin || (flags & EScanFlags::INCLUDE_BEGIN ? *begin <= key : *begin < key);
const bool matchEnd = !end || (flags & EScanFlags::INCLUDE_END ? key <= *end : key < *end);
- if (matchBegin && matchEnd) {
- if (ResolveDecommitContext.DecommitBlobs.empty() && ++NumKeysRead == maxKeys) {
- // we have hit the MaxItems limit, exit
- return true;
- }
- } else if (flags & EScanFlags::REVERSE ? !matchBegin : !matchEnd) {
- // we have exceeded the opposite boundary, exit
- return true;
+ if (matchBegin && matchEnd && ResolveDecommitContext.DecommitBlobs.empty() && ++NumKeysRead == maxKeys) {
+ // we have hit the MaxItems limit, exit
+ done = true;
+ }
+ if (flags & EScanFlags::REVERSE ? !matchEnd : !matchBegin) {
+ // we have exceeded our range
+ done = true;
}
}
}
@@ -346,7 +334,18 @@ namespace NKikimr::NBlobDepot {
? applyBegin(x.Reverse())
: applyBegin(std::forward<std::decay_t<decltype(x)>>(x));
};
- if (applyReverse(db.Table<Schema::Data>())) {
+
+ const bool status = applyReverse(db.Table<Schema::Data>());
+
+ if (boundary) {
+ if (flags & EScanFlags::REVERSE) {
+ Self->Data->LoadedKeys |= {*boundary, end.value_or(TKey::Max())};
+ } else {
+ Self->Data->LoadedKeys |= {begin.value_or(TKey::Min()), *boundary};
+ }
+ }
+
+ if (status) {
continue; // all work done for this item
} else if (progress) {
// we have already done something, so let's finish this transaction and start a new one, continuing
diff --git a/ydb/core/blob_depot/garbage_collection.cpp b/ydb/core/blob_depot/garbage_collection.cpp
index 9994114bc3..4aa554fa66 100644
--- a/ydb/core/blob_depot/garbage_collection.cpp
+++ b/ydb/core/blob_depot/garbage_collection.cpp
@@ -184,8 +184,6 @@ namespace NKikimr::NBlobDepot {
.HardGenCtr = hardGenCtr,
.Hard = hard,
};
-
- Self->BarrierServer->ValidateBlobInvariant(tabletId, channel);
}
bool TBlobDepot::TBarrierServer::AddBarrierOnDecommit(const TEvBlobStorage::TEvAssimilateResult::TBarrier& barrier,
diff --git a/ydb/core/blob_depot/given_id_range_ut.cpp b/ydb/core/blob_depot/given_id_range_ut.cpp
new file mode 100644
index 0000000000..f7739134fb
--- /dev/null
+++ b/ydb/core/blob_depot/given_id_range_ut.cpp
@@ -0,0 +1,177 @@
+#include "types.h"
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NKikimr::NBlobDepot;
+
+ui32 GenerateRandomValue(ui32 min, ui32 max) {
+ return min + RandomNumber(max - min + 1);
+}
+
+TGivenIdRange GenerateRandomRange(ui32 maxItems) {
+ TGivenIdRange res;
+
+ for (ui32 issuePos = 0; issuePos != maxItems; ) {
+ const bool value = RandomNumber(2u);
+ const ui32 numItems = GenerateRandomValue(1, maxItems - issuePos);
+ if (value) {
+ res.IssueNewRange(issuePos, issuePos + numItems);
+ }
+ issuePos += numItems;
+ }
+
+ res.CheckConsistency();
+ return res;
+}
+
+TGivenIdRange RangeFromArray(const std::vector<bool>& array) {
+ TGivenIdRange res;
+
+ std::optional<ui32> sequenceBegin;
+ for (ui32 i = 0; i < array.size(); ++i) {
+ if (array[i]) {
+ if (!sequenceBegin) {
+ sequenceBegin = i;
+ }
+ } else {
+ if (sequenceBegin) {
+ res.IssueNewRange(*sequenceBegin, i);
+ }
+ sequenceBegin.reset();
+ }
+ }
+ if (sequenceBegin) {
+ res.IssueNewRange(*sequenceBegin, array.size());
+ }
+
+ res.CheckConsistency();
+ UNIT_ASSERT_EQUAL(res.ToDebugArray(array.size()), array);
+ return res;
+}
+
+Y_UNIT_TEST_SUITE(GivenIdRange) {
+ Y_UNIT_TEST(IssueNewRange) {
+ for (ui32 i = 0; i < 1000; ++i) {
+ GenerateRandomRange(GenerateRandomValue(1, 1000));
+ }
+ }
+
+ Y_UNIT_TEST(Trim) {
+ for (ui32 i = 0; i < 1000; ++i) {
+ const ui32 maxItems = GenerateRandomValue(1, 1000);
+ TGivenIdRange range = GenerateRandomRange(maxItems);
+ TGivenIdRange originalRange(range);
+ const ui32 validSince = GenerateRandomValue(0, maxItems);
+ TGivenIdRange trimmed = range.Trim(validSince);
+ trimmed.CheckConsistency();
+ std::vector<bool> originalRangeArr = originalRange.ToDebugArray(maxItems);
+ std::vector<bool> rangeArr = range.ToDebugArray(maxItems);
+ std::vector<bool> trimmedArr = trimmed.ToDebugArray(maxItems);
+ for (ui32 i = 0; i < maxItems; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(originalRangeArr[i], rangeArr[i] + trimmedArr[i]);
+ if (i < validSince) {
+ UNIT_ASSERT(!rangeArr[i]);
+ } else {
+ UNIT_ASSERT(!trimmedArr[i]);
+ }
+ }
+ }
+ }
+
+ Y_UNIT_TEST(Subtract) {
+ for (ui32 i = 0; i < 10000; ++i) {
+ const ui32 maxItems = GenerateRandomValue(1, 1000);
+ TGivenIdRange range = GenerateRandomRange(maxItems);
+ std::vector<bool> rangeArr = range.ToDebugArray(maxItems);
+ std::vector<bool> subtractArr = GenerateRandomRange(maxItems).ToDebugArray(maxItems);
+ std::vector<bool> resArr = rangeArr;
+ for (ui32 i = 0; i < rangeArr.size(); ++i) {
+ if (subtractArr[i]) {
+ if (rangeArr[i]) {
+ resArr[i] = false;
+ } else {
+ subtractArr[i] = false;
+ }
+ }
+ }
+ TGivenIdRange subtract = RangeFromArray(subtractArr);
+ range.Subtract(subtract);
+ range.CheckConsistency();
+ UNIT_ASSERT_EQUAL(range.ToDebugArray(maxItems), resArr);
+ }
+ }
+
+ Y_UNIT_TEST(Points) {
+ for (ui32 i = 0; i < 100; ++i) {
+ const ui32 maxItems = GenerateRandomValue(1, 1000);
+ std::vector<bool> items(maxItems, false);
+ TGivenIdRange range;
+
+ for (ui32 j = 0; j < 1000; ++j) {
+ const ui32 index = RandomNumber(maxItems);
+ if (items[index]) {
+ range.RemovePoint(index);
+ } else {
+ range.AddPoint(index);
+ }
+ items[index] = !items[index];
+ range.CheckConsistency();
+ UNIT_ASSERT_EQUAL(range.ToDebugArray(maxItems), items);
+ }
+
+ for (ui32 i = 0; i < maxItems; ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(range.GetPoint(i), items[i]);
+ }
+
+ const size_t index = std::find(items.begin(), items.end(), true) - items.begin();
+ if (index != items.size()) {
+ UNIT_ASSERT(!range.IsEmpty());
+ UNIT_ASSERT_VALUES_EQUAL(range.GetMinimumValue(), index);
+ } else {
+ UNIT_ASSERT(range.IsEmpty());
+ }
+ }
+ }
+
+ Y_UNIT_TEST(Runs) {
+ for (ui32 i = 0; i < 100; ++i) {
+ const ui32 maxItems = GenerateRandomValue(1, 1000);
+ std::vector<bool> items(maxItems, false);
+ TGivenIdRange range;
+
+ for (ui32 j = 0; j < 100; ++j) {
+ const ui32 index = RandomNumber(maxItems);
+ const bool value = items[index];
+ ui32 maxRunLen = 0;
+ while (index + maxRunLen < maxItems && items[index + maxRunLen] == value) {
+ ++maxRunLen;
+ }
+ const ui32 runLen = GenerateRandomValue(1, maxRunLen);
+ std::fill(items.begin() + index, items.begin() + index + runLen, !value);
+ if (value) {
+ for (ui32 i = 0; i < runLen; ++i) {
+ range.RemovePoint(index + i);
+ }
+ } else {
+ range.IssueNewRange(index, index + runLen);
+ }
+ range.CheckConsistency();
+ UNIT_ASSERT_EQUAL(range.ToDebugArray(maxItems), items);
+ }
+ }
+ }
+
+ Y_UNIT_TEST(Allocate) {
+ for (ui32 i = 0; i < 100; ++i) {
+ const ui32 maxItems = GenerateRandomValue(1, 1000);
+ TGivenIdRange range = GenerateRandomRange(maxItems);
+ std::vector<bool> items = range.ToDebugArray(maxItems);
+ while (!range.IsEmpty()) {
+ const ui32 index = range.Allocate();
+ UNIT_ASSERT_EQUAL(items.begin() + index, std::find(items.begin(), items.end(), true));
+ items[index] = false;
+ range.CheckConsistency();
+ UNIT_ASSERT_EQUAL(range.ToDebugArray(maxItems), items);
+ }
+ }
+ }
+}
diff --git a/ydb/core/blob_depot/op_commit_blob_seq.cpp b/ydb/core/blob_depot/op_commit_blob_seq.cpp
index 964e0e90b1..e062d00f30 100644
--- a/ydb/core/blob_depot/op_commit_blob_seq.cpp
+++ b/ydb/core/blob_depot/op_commit_blob_seq.cpp
@@ -143,11 +143,12 @@ namespace NKikimr::NBlobDepot {
auto row = db.Table<Table>().Key(item.GetKey()).Select();
if (!row.IsReady()) {
success = false;
- } else if (row.IsValid()) {
- Self->Data->AddDataOnLoad(std::move(key), row.GetValue<Table::Value>(),
- row.GetValueOrDefault<Table::UncertainWrite>(), true);
} else {
- Self->Data->AddLoadSkip(std::move(key));
+ Self->Data->LoadedKeys |= {key, key};
+ if (row.IsValid()) {
+ Self->Data->AddDataOnLoad(std::move(key), row.GetValue<Table::Value>(),
+ row.GetValueOrDefault<Table::UncertainWrite>());
+ }
}
}
return success;
diff --git a/ydb/core/blob_depot/ut/CMakeLists.darwin.txt b/ydb/core/blob_depot/ut/CMakeLists.darwin.txt
new file mode 100644
index 0000000000..771f7f20fc
--- /dev/null
+++ b/ydb/core/blob_depot/ut/CMakeLists.darwin.txt
@@ -0,0 +1,66 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-blob_depot-ut)
+target_include_directories(ydb-core-blob_depot-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot
+)
+target_link_libraries(ydb-core-blob_depot-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ ydb-core-blob_depot
+)
+target_link_options(ydb-core-blob_depot-ut PRIVATE
+ -Wl,-no_deduplicate
+ -Wl,-sdk_version,10.15
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-core-blob_depot-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot/closed_interval_set_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot/given_id_range_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-core-blob_depot-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-core-blob_depot-ut
+ TEST_TARGET
+ ydb-core-blob_depot-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-blob_depot-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-blob_depot-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+vcs_info(ydb-core-blob_depot-ut)
diff --git a/ydb/core/blob_depot/ut/CMakeLists.linux-aarch64.txt b/ydb/core/blob_depot/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 0000000000..c756f2a4f0
--- /dev/null
+++ b/ydb/core/blob_depot/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-blob_depot-ut)
+target_include_directories(ydb-core-blob_depot-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot
+)
+target_link_libraries(ydb-core-blob_depot-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-lfalloc
+ cpp-testing-unittest_main
+ ydb-core-blob_depot
+)
+target_link_options(ydb-core-blob_depot-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-blob_depot-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot/closed_interval_set_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot/given_id_range_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-core-blob_depot-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-core-blob_depot-ut
+ TEST_TARGET
+ ydb-core-blob_depot-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-blob_depot-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-blob_depot-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+vcs_info(ydb-core-blob_depot-ut)
diff --git a/ydb/core/blob_depot/ut/CMakeLists.linux.txt b/ydb/core/blob_depot/ut/CMakeLists.linux.txt
new file mode 100644
index 0000000000..284e9732b5
--- /dev/null
+++ b/ydb/core/blob_depot/ut/CMakeLists.linux.txt
@@ -0,0 +1,71 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-core-blob_depot-ut)
+target_include_directories(ydb-core-blob_depot-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot
+)
+target_link_libraries(ydb-core-blob_depot-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ ydb-core-blob_depot
+)
+target_link_options(ydb-core-blob_depot-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-core-blob_depot-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot/closed_interval_set_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/blob_depot/given_id_range_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-core-blob_depot-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-core-blob_depot-ut
+ TEST_TARGET
+ ydb-core-blob_depot-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-core-blob_depot-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-core-blob_depot-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+vcs_info(ydb-core-blob_depot-ut)
diff --git a/ydb/core/blob_depot/ut/CMakeLists.txt b/ydb/core/blob_depot/ut/CMakeLists.txt
new file mode 100644
index 0000000000..bede1861df
--- /dev/null
+++ b/ydb/core/blob_depot/ut/CMakeLists.txt
@@ -0,0 +1,15 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (APPLE)
+ include(CMakeLists.darwin.txt)
+elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND UNIX AND NOT APPLE AND NOT ANDROID)
+ include(CMakeLists.linux.txt)
+endif()