aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorandrew-rykov <arykov@ydb.tech>2022-11-21 12:45:11 +0300
committerandrew-rykov <arykov@ydb.tech>2022-11-21 12:45:11 +0300
commit6b13e9638c8fa795d4c1f1e8036eee2beb8517d6 (patch)
tree7b5949f3e14cfcbf530517fda3e4c31ae4649881
parent0b5edaa9cfe1510fd2d128fd8b5383885f9a6bf9 (diff)
downloadydb-6b13e9638c8fa795d4c1f1e8036eee2beb8517d6.tar.gz
audit log system time
-rw-r--r--ydb/core/audit/audit_log.cpp11
-rw-r--r--ydb/core/audit/audit_log.h4
-rw-r--r--ydb/core/audit/audit_log_json_impl.cpp8
-rw-r--r--ydb/core/audit/audit_log_txt_impl.cpp6
-rw-r--r--ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp319
-rw-r--r--ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp160
6 files changed, 6 insertions, 502 deletions
diff --git a/ydb/core/audit/audit_log.cpp b/ydb/core/audit/audit_log.cpp
index d6a0a23445b..a9f95599a00 100644
--- a/ydb/core/audit/audit_log.cpp
+++ b/ydb/core/audit/audit_log.cpp
@@ -15,7 +15,7 @@ THolder<NActors::IActor> CreateAuditWriter(THolder<TLogBackend> auditFile, NKiki
{
AUDIT_LOG_ENABLED.store(true);
switch (format) {
- case NKikimrConfig::TAuditConfig::JSON:
+ case NKikimrConfig::TAuditConfig::JSON:
return MakeHolder<TAuditJsonLogActor>(std::move(auditFile));
case NKikimrConfig::TAuditConfig::TXT:
return MakeHolder<TAuditTxtLogActor>(std::move(auditFile));
@@ -24,19 +24,10 @@ THolder<NActors::IActor> CreateAuditWriter(THolder<TLogBackend> auditFile, NKiki
}
}
-
void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TStringBuf, TString>>& parts)
{
auto request = MakeHolder<TEvAuditLog::TEvWriteAuditLog>(Now(), parts);
sys->Send(MakeAuditServiceID(), request.Release());
}
-const char* FormatLocalTimestamp(TInstant time, char* buf) {
- struct tm localTime;
- time.LocalTime(&localTime);
- int r = strftime(buf, TimeBufSize, "%Y-%m-%d-%H-%M-%S", &localTime);
- Y_VERIFY(r != 0);
- return buf;
-}
-
} // namespace NKikimr::NAudit
diff --git a/ydb/core/audit/audit_log.h b/ydb/core/audit/audit_log.h
index d5bc7e6bed3..4622a93d517 100644
--- a/ydb/core/audit/audit_log.h
+++ b/ydb/core/audit/audit_log.h
@@ -143,8 +143,4 @@ inline NActors::TActorId MakeAuditServiceID() {
THolder<NActors::IActor> CreateAuditWriter(
THolder<TLogBackend> auditFile, NKikimrConfig::TAuditConfig_EFormat format);
-constexpr size_t TimeBufSize = 512;
-
-const char* FormatLocalTimestamp(TInstant time, char* buf);
-
} // namespace NKikimr::NAudit
diff --git a/ydb/core/audit/audit_log_json_impl.cpp b/ydb/core/audit/audit_log_json_impl.cpp
index 689c5648d5a..4f84d2bd8fb 100644
--- a/ydb/core/audit/audit_log_json_impl.cpp
+++ b/ydb/core/audit/audit_log_json_impl.cpp
@@ -31,11 +31,9 @@ STFUNC(TAuditJsonLogActor::StateWork)
void TAuditJsonLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, const TActorContext& ctx) {
Y_UNUSED(ctx);
const auto* msg = ev->Get();
- try {
+ try {
TStringStream ss;
- char buf[TimeBufSize];
- ss << FormatLocalTimestamp(msg->Time, buf) << ": ";
-
+ ss << msg->Time << ": ";
NJson::TJsonMap m;
for (auto& [k, v] : msg->Parts) {
m[k] = v;
@@ -43,7 +41,7 @@ void TAuditJsonLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog
NJson::WriteJson(&ss, &m, false, false);
ss << Endl;
auto json = ss.Str();
-
+
AuditFile->WriteData(
TLogRecord(
ELogPriority::TLOG_INFO,
diff --git a/ydb/core/audit/audit_log_txt_impl.cpp b/ydb/core/audit/audit_log_txt_impl.cpp
index 972dadfb03e..5f25119cc77 100644
--- a/ydb/core/audit/audit_log_txt_impl.cpp
+++ b/ydb/core/audit/audit_log_txt_impl.cpp
@@ -30,9 +30,7 @@ void TAuditTxtLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog:
const auto* msg = ev->Get();
try {
TStringStream ss;
- char buf[TimeBufSize];
- ss << FormatLocalTimestamp(msg->Time, buf) << ": ";
-
+ ss << msg->Time << ": ";
for (auto it = msg->Parts.begin(); it != msg->Parts.end(); it++) {
if (it != msg->Parts.begin())
ss << ", ";
@@ -40,7 +38,7 @@ void TAuditTxtLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog:
}
ss << Endl;
auto text = ss.Str();
-
+
AuditFile->WriteData(
TLogRecord(
ELogPriority::TLOG_INFO,
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp
deleted file mode 100644
index 9147c6a8715..00000000000
--- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp
+++ /dev/null
@@ -1,319 +0,0 @@
-#include <library/cpp/testing/unittest/registar.h>
-#include <util/random/fast.h>
-#include "blobstorage_groupinfo_blobmap.h"
-#include "blobstorage_groupinfo_iter.h"
-
-using namespace NKikimr;
-
-namespace {
-
- struct TOriginalBlobStorageGroupInfo {
- ui32 BlobSubgroupSize;
- TVector<TBlobStorageGroupInfo::TFailRealm> Realms;
- const ui32 GroupID;
- const ui32 GroupGeneration;
- TBlobStorageGroupInfo::TDynamicInfo DynamicInfo;
-
- TOriginalBlobStorageGroupInfo(ui32 blobSubgroupSize, const TBlobStorageGroupInfo& groupInfo)
- : BlobSubgroupSize(blobSubgroupSize)
- , GroupID(groupInfo.GroupID)
- , GroupGeneration(groupInfo.GroupGeneration)
- , DynamicInfo(groupInfo.GetDynamicInfo())
- {
- for (auto it = groupInfo.FailRealmsBegin(); it != groupInfo.FailRealmsEnd(); ++it) {
- Realms.push_back(*it);
- }
- }
-
- bool PickSubgroup(ui32 hash, ui32 vdiskSz, TVDiskID *outVDisk, TActorId *outServiceIds) const {
- const ui32 subgroupSz = BlobSubgroupSize;
- Y_VERIFY(vdiskSz == subgroupSz);
-
- const TBlobStorageGroupInfo::TFailRealm& realm = Realms[0];
- Y_VERIFY_DEBUG(realm.FailDomains.size() >= subgroupSz);
-
- ui32 domainIdx = hash % realm.FailDomains.size();
- TReallyFastRng32 rng(hash);
-
- for (ui32 i = 0; i < subgroupSz; ++i) {
- const ui32 dx = domainIdx % realm.FailDomains.size();
- const TBlobStorageGroupInfo::TFailDomain &domain = realm.FailDomains[dx];
- const ui32 vx = rng() % domain.VDisks.size();
-
- outVDisk[i] = TVDiskID(GroupID, GroupGeneration, 0, dx, vx);
- auto orderNum = domain.VDisks[vx].OrderNumber;
- outServiceIds[i] = DynamicInfo.ServiceIdForOrderNumber[orderNum];
-
- ++domainIdx;
- }
-
- return true;
- }
-
- bool BelongsToSubgroup(const TVDiskID &vdisk, ui32 hash) const {
- bool isReplica = GetIdxInSubgroup(vdisk, hash) < BlobSubgroupSize;
- return isReplica;
- }
-
- // Returns either vdisk idx in the blob subgroup, or BlobSubgroupSize if the vdisk is not in the blob subgroup
- ui32 GetIdxInSubgroup(const TVDiskID &vdisk, ui32 hash) const {
- Y_VERIFY(vdisk.GroupID == GroupID && vdisk.GroupGeneration == GroupGeneration);
- Y_VERIFY(vdisk.FailRealm < Realms.size());
-
- const TBlobStorageGroupInfo::TFailRealm &realm = Realms[vdisk.FailRealm];
- ui32 idx = (vdisk.FailDomain + realm.FailDomains.size() - hash % realm.FailDomains.size()) % realm.FailDomains.size();
-
- if (idx < BlobSubgroupSize) {
- ui32 vdiskIdx;
- TReallyFastRng32 rng(hash);
- for (ui32 i = 0; i < idx; ++i)
- rng();
- vdiskIdx = rng() % realm.FailDomains[vdisk.FailDomain].VDisks.size();
- if (vdiskIdx == vdisk.VDisk) {
- return idx;
- }
- }
- return BlobSubgroupSize;
- }
-
- TVDiskID GetVDiskInSubgroup(ui32 idxInSubgroup, ui32 hash) const {
- Y_VERIFY(idxInSubgroup < BlobSubgroupSize);
-
- const TBlobStorageGroupInfo::TFailRealm &xrealm = Realms[0];
- ui32 domainIdx = hash % xrealm.FailDomains.size() + idxInSubgroup;
- const ui32 dx = domainIdx % xrealm.FailDomains.size();
- const TBlobStorageGroupInfo::TFailDomain &domain = xrealm.FailDomains[dx];
- TReallyFastRng32 rng(hash);
- for (ui32 i = 0; i < idxInSubgroup; ++i) {
- rng();
- }
- const ui32 vx = rng() % domain.VDisks.size();
-
- return TVDiskID(GroupID, GroupGeneration, 0, dx, vx);
- }
- };
-
-} // anon
-
-Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) {
-
- void MakeBelongsToSubgroupBenchmark(TBlobStorageGroupType::EErasureSpecies erasure, ui32 numFailDomains,
- NUnitTest::TTestContext& ut_context) {
- auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(erasure, 1, numFailDomains, 1);
- const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize();
- TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo);
-
- TVector<TLogoBlobID> ids;
- for (ui32 i = 0; i < 10000; ++i) {
- for (ui32 j = 0; j < 1000; ++j) {
- ids.emplace_back(i, 1, j, 1, 1000, 1);
- }
- }
-
- constexpr ui64 blobCount = 10'000'000;
- UNIT_ASSERT(blobCount == ids.size());
- ui64 iterationCount = numFailDomains * blobCount;
-
- THPTimer timer;
- ui32 num = 0;
- for (const auto& vdisk : groupInfo->GetVDisks()) {
- for (const TLogoBlobID& id : ids) {
- auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber);
- num += groupInfo->BelongsToSubgroup(vd, id.Hash()) ? 1 : 0;
- }
- }
- double newMetric = 1'000'000'000 * timer.PassedReset() / iterationCount;
- TString newMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure)
- << " domains " << numFailDomains
- << " new (ns)";
- ut_context.Metrics[newMetricsName] = newMetric;
- Cerr << newMetricsName << ": " << newMetric<< Endl;
-
- timer.Reset();
- ui32 num2 = 0;
- for (const auto& vdisk : groupInfo->GetVDisks()) {
- for (const TLogoBlobID& id : ids) {
- auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber);
- num2 += orig.BelongsToSubgroup(vd, id.Hash()) ? 1 : 0;
- }
- }
- double oldMetric = 1'000'000'000 * timer.PassedReset() / iterationCount;
- TString oldMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure)
- << " domains " << numFailDomains
- << " old (ns)";
- ut_context.Metrics[oldMetricsName] = oldMetric;
- Cerr << oldMetricsName << ": " << oldMetric << Endl;
-
- UNIT_ASSERT_VALUES_EQUAL(num, num2);
- }
-
- Y_UNIT_TEST(BelongsToSubgroupBenchmark) {
- auto erasures = {TBlobStorageGroupType::ErasureNone,
- TBlobStorageGroupType::ErasureMirror3,
- TBlobStorageGroupType::Erasure4Plus2Block,
- TBlobStorageGroupType::ErasureMirror3of4};
- for (auto erasure : erasures) {
- TBlobStorageGroupType type(erasure);
- for (ui32 domains : {type.BlobSubgroupSize(), 9u}) {
- MakeBelongsToSubgroupBenchmark(erasure, domains, ut_context);
- }
- }
- }
-
- void BasicCheck(const std::unique_ptr<TBlobStorageGroupInfo> &groupInfo, TOriginalBlobStorageGroupInfo &orig,
- TLogoBlobID id, ui32 blobSubgroupSize) {
- std::array<TVDiskID, 8> vdisks;
- std::array<TActorId, 8> services;
- orig.PickSubgroup(id.Hash(), blobSubgroupSize, vdisks.data(), services.data());
-
- TBlobStorageGroupInfo::TVDiskIds vdisks2;
- TBlobStorageGroupInfo::TServiceIds services2;
- groupInfo->PickSubgroup(id.Hash(), &vdisks2, &services2);
-
- UNIT_ASSERT_EQUAL(vdisks2.size(), blobSubgroupSize);
- UNIT_ASSERT_EQUAL(services2.size(), blobSubgroupSize);
- UNIT_ASSERT(std::equal(vdisks2.begin(), vdisks2.end(), vdisks.begin()));
- UNIT_ASSERT(std::equal(services2.begin(), services2.end(), services.begin()));
-
- for (ui32 i = 0; i < blobSubgroupSize; ++i) {
- const TVDiskID& vdisk = vdisks[i];
-
- UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()),
- orig.GetVDiskInSubgroup(i, id.Hash()));
- UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), vdisk);
-
- UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()),
- orig.GetIdxInSubgroup(vdisk, id.Hash()));
- UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), i);
- }
-
- THashMap<TVDiskID, ui32> disk2index;
- for (ui32 i = 0; i < blobSubgroupSize; ++i) {
- disk2index[vdisks2[i]] = i;
- }
-
- for (const auto& vdisk : groupInfo->GetVDisks()) {
- auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber);
- auto it = disk2index.find(vd);
- bool isReplicaFor = it != disk2index.end();
-
- UNIT_ASSERT_VALUES_EQUAL(orig.BelongsToSubgroup(vd, id.Hash()), isReplicaFor);
- UNIT_ASSERT_VALUES_EQUAL(groupInfo->BelongsToSubgroup(vd, id.Hash()), isReplicaFor);
-
- const ui32 index = isReplicaFor ? it->second : blobSubgroupSize;
- UNIT_ASSERT_VALUES_EQUAL(orig.GetIdxInSubgroup(vd, id.Hash()), index);
- UNIT_ASSERT_VALUES_EQUAL(groupInfo->GetIdxInSubgroup(vd, id.Hash()), index);
- }
- }
-
- Y_UNIT_TEST(BasicChecks) {
- for (auto erasure : {TBlobStorageGroupType::ErasureNone, TBlobStorageGroupType::ErasureMirror3,
- TBlobStorageGroupType::Erasure3Plus1Block, TBlobStorageGroupType::Erasure3Plus1Stripe,
- TBlobStorageGroupType::Erasure4Plus2Block, TBlobStorageGroupType::Erasure3Plus2Block,
- TBlobStorageGroupType::Erasure4Plus2Stripe, TBlobStorageGroupType::Erasure3Plus2Stripe,
- TBlobStorageGroupType::ErasureMirror3Plus2}) {
- auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(erasure, 3U, 8U, 1U);
- const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize();
- TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo);
-
- for (ui32 i = 0; i < 100; ++i) {
- for (ui32 j = 0; j < 10; ++j) {
- TLogoBlobID id(i, 1, j, 1, 1000, 1);
- BasicCheck(groupInfo, orig, id, blobSubgroupSize);
- }
- }
- }
- }
-
- Y_UNIT_TEST(CheckCorrectBehaviourWithHashOverlow) {
- auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(TErasureType::ErasureMirror3, 1U, 5U, 1U);
- const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize();
- TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo);
- TLogoBlobID id(4550843067551373890, 2564314201, 2840555155, 59, 0, 2230444);
- BasicCheck(groupInfo, orig, id, blobSubgroupSize);
- }
-
- Y_UNIT_TEST(Mirror3dcMapper) {
- auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(TBlobStorageGroupType::ErasureMirror3, 3U, 5U, 4U);
-
- std::unique_ptr<IBlobToDiskMapper> mapper{IBlobToDiskMapper::CreateMirror3dcMapper(&groupInfo->GetTopology())};
-
- THashMap<TVDiskID, TVector<ui32>> usageMap;
- for (const auto& vdisk : groupInfo->GetVDisks()) {
- auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber);
- usageMap.emplace(vd, TVector<ui32>(9));
- }
-
- TReallyFastRng32 rng(1);
- for (ui32 i = 0; i < 10000; ++i) {
- const ui32 hash = rng();
-
- TBlobStorageGroupInfo::TOrderNums orderNums;
- TBlobStorageGroupInfo::TVDiskIds vdisks;
- mapper->PickSubgroup(hash, orderNums);
- for (const auto &x : orderNums) {
- vdisks.push_back(groupInfo->GetVDiskId(x));
- }
-
- for (ui32 i = 0; i < vdisks.size(); ++i) {
- ++usageMap[vdisks[i]][i];
- }
-
- for (ui32 i = 0; i < 3; ++i) {
- // ensure that cells within a column are in the same ring
- UNIT_ASSERT_VALUES_EQUAL(vdisks[i].FailRealm, vdisks[i + 3].FailRealm);
- UNIT_ASSERT_VALUES_EQUAL(vdisks[i].FailRealm, vdisks[i + 6].FailRealm);
- // and from different fail domains
- UNIT_ASSERT(vdisks[i].FailDomain != vdisks[i + 3].FailDomain);
- UNIT_ASSERT(vdisks[i].FailDomain != vdisks[i + 6].FailDomain);
- UNIT_ASSERT(vdisks[i + 3].FailDomain != vdisks[i + 6].FailDomain);
- }
- UNIT_ASSERT(vdisks[0].FailRealm != vdisks[1].FailRealm);
- UNIT_ASSERT(vdisks[0].FailRealm != vdisks[2].FailRealm);
- UNIT_ASSERT(vdisks[1].FailRealm != vdisks[2].FailRealm);
-
- THashMap<TVDiskID, ui32> disk2index;
- for (ui32 i = 0; i < vdisks.size(); ++i) {
- disk2index[vdisks[i]] = i;
- }
-
- for (const auto& vdisk : groupInfo->GetVDisks()) {
- auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber);
- auto it = disk2index.find(vd);
- bool isReplicaFor = it != disk2index.end();
- UNIT_ASSERT_VALUES_EQUAL(mapper->GetIdxInSubgroup(vd, hash), isReplicaFor ? it->second : 9);
- UNIT_ASSERT_VALUES_EQUAL(mapper->BelongsToSubgroup(vd, hash), isReplicaFor);
- }
-
- for (ui32 i = 0; i < vdisks.size(); ++i) {
- UNIT_ASSERT_EQUAL(mapper->GetVDiskInSubgroup(i, hash), vdisks[i]);
- }
- }
-
- TVector<double> xv;
- for (const auto& kv : usageMap) {
- Cerr << kv.first.ToString() << "#";
- for (ui32 i : kv.second) {
- xv.push_back(i);
- Cerr << " " << i;
- }
- Cerr << Endl;
- }
-
- double mean = 0;
- for (double x : xv) {
- mean += x;
- }
- mean /= xv.size();
-
- double dev = 0;
- for (double x : xv) {
- double d = x - mean;
- dev += d * d;
- }
- dev = sqrt(dev / xv.size());
-
- Cerr << "mean# " << mean << " dev# " << dev << Endl;
- }
-
-}
diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp
deleted file mode 100644
index 900f40a0b7b..00000000000
--- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp
+++ /dev/null
@@ -1,160 +0,0 @@
-#include <library/cpp/testing/unittest/registar.h>
-#include "blobstorage_groupinfo_partlayout.h"
-#include <thread>
-#include <mutex>
-#include <condition_variable>
-
-using namespace NKikimr;
-
-ui32 CountEffectiveReplicas(const TSubgroupPartLayout& layout, ui32 numRows, ui32 numCols) {
- // number of effective replicas is calculated as the maximum number of rows that can be mapped to unique columns in
- // layout with bit at intersecting cell set to zero; here we state that numCols >= numRows
- UNIT_ASSERT(numCols >= numRows);
- std::vector<ui32> index(numCols);
- for (size_t i = 0; i < index.size(); ++i) {
- index[i] = i;
- }
- ui32 res = 0;
- do {
- ui32 value = 0;
- for (ui32 row = 0; row < numRows; ++row) {
- // extract the cell at row, col and count it if it is nonzero
- value += layout.GetDisksWithPart(row) >> index[row] & 1;
- }
- if (value > res) {
- res = value;
- }
- } while (std::next_permutation(index.begin(), index.end()));
- return res;
-}
-
-class TCheckQueue {
- std::mutex Mutex;
- std::list<std::thread> Threads;
- std::deque<TSubgroupPartLayout> Queue;
- std::condition_variable Cvar, ReverseCvar;
- volatile bool Done = false;
- std::atomic_size_t CasesChecked = 0;
- std::atomic_uint64_t Timing = 0;
-
-public:
- TCheckQueue(TBlobStorageGroupType gtype) {
- for (int n = std::thread::hardware_concurrency(); n; --n) {
- Threads.emplace_back([=] {
- for (;;) {
- TSubgroupPartLayout layout;
- {
- std::unique_lock<std::mutex> lock(Mutex);
- Cvar.wait(lock, [&] { return Done || !Queue.empty(); });
- if (Queue.empty() && Done) {
- return;
- }
- Y_VERIFY(!Queue.empty());
- layout = std::move(Queue.front());
- Queue.pop_front();
- if (Queue.size() < 1024) {
- ReverseCvar.notify_one();
- }
- }
-
- // count effective replicas
- THPTimer timer;
- ui32 count = layout.CountEffectiveReplicas(gtype);
- Timing += 1e9 * timer.Passed();
-
- // compare it with generic value
- ui32 generic = CountEffectiveReplicas(layout, gtype.TotalPartCount(), gtype.BlobSubgroupSize());
-
- // verify the value
- Y_VERIFY(count == generic, "count# %" PRIu32 " generic# %" PRIu32 " layout# %s erasure# %s",
- count, generic, layout.ToString(gtype).data(),
- TBlobStorageGroupType::ErasureSpeciesName(gtype.GetErasure()).data());
-
- ++CasesChecked;
- }
- });
- }
- }
-
- ~TCheckQueue() {
- {
- std::unique_lock<std::mutex> lock(Mutex);
- Done = true;
- }
- Cvar.notify_all();
- for (auto& thread : Threads) {
- thread.join();
- }
- Cerr << "Checked " << (size_t)CasesChecked << " cases, took " << (ui64)Timing / 1000 << " us" << Endl;
- }
-
- void operator ()(const TSubgroupPartLayout& layout) {
- std::unique_lock<std::mutex> lock(Mutex);
- ReverseCvar.wait(lock, [&] { return Queue.size() < 1024; });
- Queue.push_back(layout);
- Cvar.notify_one();
- }
-};
-
-void TestErasureSet(ui32 firstIdx, ui32 step) {
- for (ui32 erasure = firstIdx; erasure < TBlobStorageGroupType::ErasureSpeciesCount; erasure += step) {
- if (erasure == TBlobStorageGroupType::ErasureMirror3dc || erasure == TBlobStorageGroupType::ErasureMirror3of4) {
- continue;
- }
- TBlobStorageGroupType gtype(static_cast<TBlobStorageGroupType::EErasureSpecies>(erasure));
- if (gtype.BlobSubgroupSize() > 8) {
- continue;
- }
- Cerr << "testing erasure " << TBlobStorageGroupType::ErasureSpeciesName(erasure) << Endl;
- const ui32 totalPartCount = gtype.TotalPartCount();
- const ui32 blobSubgroupSize = gtype.BlobSubgroupSize();
- TCheckQueue checker(gtype);
- for (ui32 main = 0; main < 1 << totalPartCount; ++main) {
- Cerr << " main# " << main << Endl;
- std::vector<ui32> handoffs(gtype.Handoff(), 0);
- for (;;) {
- // generate subgroup layout
- TSubgroupPartLayout layout;
- for (ui32 nodeId = 0; nodeId < blobSubgroupSize; ++nodeId) {
- if (nodeId < totalPartCount) {
- if (main & 1 << nodeId) {
- layout.AddItem(nodeId, nodeId, gtype);
- }
- } else {
- for (ui32 i = 0; i < totalPartCount; ++i) {
- if (handoffs[nodeId - totalPartCount] & 1 << i) {
- layout.AddItem(nodeId, i, gtype);
- }
- }
- }
- }
-
- checker(layout);
-
- // increment handoffs
- ui32 carry = 1;
- for (size_t i = 0; carry && i < handoffs.size(); ++i) {
- carry = !(++handoffs[i] & (1 << totalPartCount) - 1);
- }
- if (carry) {
- break;
- }
- }
- }
- }
-}
-
-Y_UNIT_TEST_SUITE(TSubgroupPartLayoutTest) {
- Y_UNIT_TEST(CountEffectiveReplicas1of4) {
- TestErasureSet(0, 4);
- }
- Y_UNIT_TEST(CountEffectiveReplicas2of4) {
- TestErasureSet(1, 4);
- }
- Y_UNIT_TEST(CountEffectiveReplicas3of4) {
- TestErasureSet(2, 4);
- }
- Y_UNIT_TEST(CountEffectiveReplicas4of4) {
- TestErasureSet(3, 4);
- }
-}