diff options
author | andrew-rykov <arykov@ydb.tech> | 2022-11-21 12:45:11 +0300 |
---|---|---|
committer | andrew-rykov <arykov@ydb.tech> | 2022-11-21 12:45:11 +0300 |
commit | 6b13e9638c8fa795d4c1f1e8036eee2beb8517d6 (patch) | |
tree | 7b5949f3e14cfcbf530517fda3e4c31ae4649881 | |
parent | 0b5edaa9cfe1510fd2d128fd8b5383885f9a6bf9 (diff) | |
download | ydb-6b13e9638c8fa795d4c1f1e8036eee2beb8517d6.tar.gz |
audit log system time
-rw-r--r-- | ydb/core/audit/audit_log.cpp | 11 | ||||
-rw-r--r-- | ydb/core/audit/audit_log.h | 4 | ||||
-rw-r--r-- | ydb/core/audit/audit_log_json_impl.cpp | 8 | ||||
-rw-r--r-- | ydb/core/audit/audit_log_txt_impl.cpp | 6 | ||||
-rw-r--r-- | ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp | 319 | ||||
-rw-r--r-- | ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp | 160 |
6 files changed, 6 insertions, 502 deletions
diff --git a/ydb/core/audit/audit_log.cpp b/ydb/core/audit/audit_log.cpp index d6a0a23445b..a9f95599a00 100644 --- a/ydb/core/audit/audit_log.cpp +++ b/ydb/core/audit/audit_log.cpp @@ -15,7 +15,7 @@ THolder<NActors::IActor> CreateAuditWriter(THolder<TLogBackend> auditFile, NKiki { AUDIT_LOG_ENABLED.store(true); switch (format) { - case NKikimrConfig::TAuditConfig::JSON: + case NKikimrConfig::TAuditConfig::JSON: return MakeHolder<TAuditJsonLogActor>(std::move(auditFile)); case NKikimrConfig::TAuditConfig::TXT: return MakeHolder<TAuditTxtLogActor>(std::move(auditFile)); @@ -24,19 +24,10 @@ THolder<NActors::IActor> CreateAuditWriter(THolder<TLogBackend> auditFile, NKiki } } - void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TStringBuf, TString>>& parts) { auto request = MakeHolder<TEvAuditLog::TEvWriteAuditLog>(Now(), parts); sys->Send(MakeAuditServiceID(), request.Release()); } -const char* FormatLocalTimestamp(TInstant time, char* buf) { - struct tm localTime; - time.LocalTime(&localTime); - int r = strftime(buf, TimeBufSize, "%Y-%m-%d-%H-%M-%S", &localTime); - Y_VERIFY(r != 0); - return buf; -} - } // namespace NKikimr::NAudit diff --git a/ydb/core/audit/audit_log.h b/ydb/core/audit/audit_log.h index d5bc7e6bed3..4622a93d517 100644 --- a/ydb/core/audit/audit_log.h +++ b/ydb/core/audit/audit_log.h @@ -143,8 +143,4 @@ inline NActors::TActorId MakeAuditServiceID() { THolder<NActors::IActor> CreateAuditWriter( THolder<TLogBackend> auditFile, NKikimrConfig::TAuditConfig_EFormat format); -constexpr size_t TimeBufSize = 512; - -const char* FormatLocalTimestamp(TInstant time, char* buf); - } // namespace NKikimr::NAudit diff --git a/ydb/core/audit/audit_log_json_impl.cpp b/ydb/core/audit/audit_log_json_impl.cpp index 689c5648d5a..4f84d2bd8fb 100644 --- a/ydb/core/audit/audit_log_json_impl.cpp +++ b/ydb/core/audit/audit_log_json_impl.cpp @@ -31,11 +31,9 @@ STFUNC(TAuditJsonLogActor::StateWork) void TAuditJsonLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ctx); const auto* msg = ev->Get(); - try { + try { TStringStream ss; - char buf[TimeBufSize]; - ss << FormatLocalTimestamp(msg->Time, buf) << ": "; - + ss << msg->Time << ": "; NJson::TJsonMap m; for (auto& [k, v] : msg->Parts) { m[k] = v; @@ -43,7 +41,7 @@ void TAuditJsonLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog NJson::WriteJson(&ss, &m, false, false); ss << Endl; auto json = ss.Str(); - + AuditFile->WriteData( TLogRecord( ELogPriority::TLOG_INFO, diff --git a/ydb/core/audit/audit_log_txt_impl.cpp b/ydb/core/audit/audit_log_txt_impl.cpp index 972dadfb03e..5f25119cc77 100644 --- a/ydb/core/audit/audit_log_txt_impl.cpp +++ b/ydb/core/audit/audit_log_txt_impl.cpp @@ -30,9 +30,7 @@ void TAuditTxtLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog: const auto* msg = ev->Get(); try { TStringStream ss; - char buf[TimeBufSize]; - ss << FormatLocalTimestamp(msg->Time, buf) << ": "; - + ss << msg->Time << ": "; for (auto it = msg->Parts.begin(); it != msg->Parts.end(); it++) { if (it != msg->Parts.begin()) ss << ", "; @@ -40,7 +38,7 @@ void TAuditTxtLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog: } ss << Endl; auto text = ss.Str(); - + AuditFile->WriteData( TLogRecord( ELogPriority::TLOG_INFO, diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp deleted file mode 100644 index 9147c6a8715..00000000000 --- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_blobmap_ut.cpp +++ /dev/null @@ -1,319 +0,0 @@ -#include <library/cpp/testing/unittest/registar.h> -#include <util/random/fast.h> -#include "blobstorage_groupinfo_blobmap.h" -#include "blobstorage_groupinfo_iter.h" - -using namespace NKikimr; - -namespace { - - struct TOriginalBlobStorageGroupInfo { - ui32 BlobSubgroupSize; - TVector<TBlobStorageGroupInfo::TFailRealm> Realms; - const ui32 GroupID; - const ui32 GroupGeneration; - TBlobStorageGroupInfo::TDynamicInfo DynamicInfo; - - TOriginalBlobStorageGroupInfo(ui32 blobSubgroupSize, const TBlobStorageGroupInfo& groupInfo) - : BlobSubgroupSize(blobSubgroupSize) - , GroupID(groupInfo.GroupID) - , GroupGeneration(groupInfo.GroupGeneration) - , DynamicInfo(groupInfo.GetDynamicInfo()) - { - for (auto it = groupInfo.FailRealmsBegin(); it != groupInfo.FailRealmsEnd(); ++it) { - Realms.push_back(*it); - } - } - - bool PickSubgroup(ui32 hash, ui32 vdiskSz, TVDiskID *outVDisk, TActorId *outServiceIds) const { - const ui32 subgroupSz = BlobSubgroupSize; - Y_VERIFY(vdiskSz == subgroupSz); - - const TBlobStorageGroupInfo::TFailRealm& realm = Realms[0]; - Y_VERIFY_DEBUG(realm.FailDomains.size() >= subgroupSz); - - ui32 domainIdx = hash % realm.FailDomains.size(); - TReallyFastRng32 rng(hash); - - for (ui32 i = 0; i < subgroupSz; ++i) { - const ui32 dx = domainIdx % realm.FailDomains.size(); - const TBlobStorageGroupInfo::TFailDomain &domain = realm.FailDomains[dx]; - const ui32 vx = rng() % domain.VDisks.size(); - - outVDisk[i] = TVDiskID(GroupID, GroupGeneration, 0, dx, vx); - auto orderNum = domain.VDisks[vx].OrderNumber; - outServiceIds[i] = DynamicInfo.ServiceIdForOrderNumber[orderNum]; - - ++domainIdx; - } - - return true; - } - - bool BelongsToSubgroup(const TVDiskID &vdisk, ui32 hash) const { - bool isReplica = GetIdxInSubgroup(vdisk, hash) < BlobSubgroupSize; - return isReplica; - } - - // Returns either vdisk idx in the blob subgroup, or BlobSubgroupSize if the vdisk is not in the blob subgroup - ui32 GetIdxInSubgroup(const TVDiskID &vdisk, ui32 hash) const { - Y_VERIFY(vdisk.GroupID == GroupID && vdisk.GroupGeneration == GroupGeneration); - Y_VERIFY(vdisk.FailRealm < Realms.size()); - - const TBlobStorageGroupInfo::TFailRealm &realm = Realms[vdisk.FailRealm]; - ui32 idx = (vdisk.FailDomain + realm.FailDomains.size() - hash % realm.FailDomains.size()) % realm.FailDomains.size(); - - if (idx < BlobSubgroupSize) { - ui32 vdiskIdx; - TReallyFastRng32 rng(hash); - for (ui32 i = 0; i < idx; ++i) - rng(); - vdiskIdx = rng() % realm.FailDomains[vdisk.FailDomain].VDisks.size(); - if (vdiskIdx == vdisk.VDisk) { - return idx; - } - } - return BlobSubgroupSize; - } - - TVDiskID GetVDiskInSubgroup(ui32 idxInSubgroup, ui32 hash) const { - Y_VERIFY(idxInSubgroup < BlobSubgroupSize); - - const TBlobStorageGroupInfo::TFailRealm &xrealm = Realms[0]; - ui32 domainIdx = hash % xrealm.FailDomains.size() + idxInSubgroup; - const ui32 dx = domainIdx % xrealm.FailDomains.size(); - const TBlobStorageGroupInfo::TFailDomain &domain = xrealm.FailDomains[dx]; - TReallyFastRng32 rng(hash); - for (ui32 i = 0; i < idxInSubgroup; ++i) { - rng(); - } - const ui32 vx = rng() % domain.VDisks.size(); - - return TVDiskID(GroupID, GroupGeneration, 0, dx, vx); - } - }; - -} // anon - -Y_UNIT_TEST_SUITE(TBlobStorageGroupInfoBlobMapTest) { - - void MakeBelongsToSubgroupBenchmark(TBlobStorageGroupType::EErasureSpecies erasure, ui32 numFailDomains, - NUnitTest::TTestContext& ut_context) { - auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(erasure, 1, numFailDomains, 1); - const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize(); - TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo); - - TVector<TLogoBlobID> ids; - for (ui32 i = 0; i < 10000; ++i) { - for (ui32 j = 0; j < 1000; ++j) { - ids.emplace_back(i, 1, j, 1, 1000, 1); - } - } - - constexpr ui64 blobCount = 10'000'000; - UNIT_ASSERT(blobCount == ids.size()); - ui64 iterationCount = numFailDomains * blobCount; - - THPTimer timer; - ui32 num = 0; - for (const auto& vdisk : groupInfo->GetVDisks()) { - for (const TLogoBlobID& id : ids) { - auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); - num += groupInfo->BelongsToSubgroup(vd, id.Hash()) ? 1 : 0; - } - } - double newMetric = 1'000'000'000 * timer.PassedReset() / iterationCount; - TString newMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure) - << " domains " << numFailDomains - << " new (ns)"; - ut_context.Metrics[newMetricsName] = newMetric; - Cerr << newMetricsName << ": " << newMetric<< Endl; - - timer.Reset(); - ui32 num2 = 0; - for (const auto& vdisk : groupInfo->GetVDisks()) { - for (const TLogoBlobID& id : ids) { - auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); - num2 += orig.BelongsToSubgroup(vd, id.Hash()) ? 1 : 0; - } - } - double oldMetric = 1'000'000'000 * timer.PassedReset() / iterationCount; - TString oldMetricsName = TStringBuilder() << TErasureType::ErasureSpeciesToStr(erasure) - << " domains " << numFailDomains - << " old (ns)"; - ut_context.Metrics[oldMetricsName] = oldMetric; - Cerr << oldMetricsName << ": " << oldMetric << Endl; - - UNIT_ASSERT_VALUES_EQUAL(num, num2); - } - - Y_UNIT_TEST(BelongsToSubgroupBenchmark) { - auto erasures = {TBlobStorageGroupType::ErasureNone, - TBlobStorageGroupType::ErasureMirror3, - TBlobStorageGroupType::Erasure4Plus2Block, - TBlobStorageGroupType::ErasureMirror3of4}; - for (auto erasure : erasures) { - TBlobStorageGroupType type(erasure); - for (ui32 domains : {type.BlobSubgroupSize(), 9u}) { - MakeBelongsToSubgroupBenchmark(erasure, domains, ut_context); - } - } - } - - void BasicCheck(const std::unique_ptr<TBlobStorageGroupInfo> &groupInfo, TOriginalBlobStorageGroupInfo &orig, - TLogoBlobID id, ui32 blobSubgroupSize) { - std::array<TVDiskID, 8> vdisks; - std::array<TActorId, 8> services; - orig.PickSubgroup(id.Hash(), blobSubgroupSize, vdisks.data(), services.data()); - - TBlobStorageGroupInfo::TVDiskIds vdisks2; - TBlobStorageGroupInfo::TServiceIds services2; - groupInfo->PickSubgroup(id.Hash(), &vdisks2, &services2); - - UNIT_ASSERT_EQUAL(vdisks2.size(), blobSubgroupSize); - UNIT_ASSERT_EQUAL(services2.size(), blobSubgroupSize); - UNIT_ASSERT(std::equal(vdisks2.begin(), vdisks2.end(), vdisks.begin())); - UNIT_ASSERT(std::equal(services2.begin(), services2.end(), services.begin())); - - for (ui32 i = 0; i < blobSubgroupSize; ++i) { - const TVDiskID& vdisk = vdisks[i]; - - UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), - orig.GetVDiskInSubgroup(i, id.Hash())); - UNIT_ASSERT_EQUAL(groupInfo->GetVDiskInSubgroup(i, id.Hash()), vdisk); - - UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), - orig.GetIdxInSubgroup(vdisk, id.Hash())); - UNIT_ASSERT_EQUAL(groupInfo->GetIdxInSubgroup(vdisk, id.Hash()), i); - } - - THashMap<TVDiskID, ui32> disk2index; - for (ui32 i = 0; i < blobSubgroupSize; ++i) { - disk2index[vdisks2[i]] = i; - } - - for (const auto& vdisk : groupInfo->GetVDisks()) { - auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); - auto it = disk2index.find(vd); - bool isReplicaFor = it != disk2index.end(); - - UNIT_ASSERT_VALUES_EQUAL(orig.BelongsToSubgroup(vd, id.Hash()), isReplicaFor); - UNIT_ASSERT_VALUES_EQUAL(groupInfo->BelongsToSubgroup(vd, id.Hash()), isReplicaFor); - - const ui32 index = isReplicaFor ? it->second : blobSubgroupSize; - UNIT_ASSERT_VALUES_EQUAL(orig.GetIdxInSubgroup(vd, id.Hash()), index); - UNIT_ASSERT_VALUES_EQUAL(groupInfo->GetIdxInSubgroup(vd, id.Hash()), index); - } - } - - Y_UNIT_TEST(BasicChecks) { - for (auto erasure : {TBlobStorageGroupType::ErasureNone, TBlobStorageGroupType::ErasureMirror3, - TBlobStorageGroupType::Erasure3Plus1Block, TBlobStorageGroupType::Erasure3Plus1Stripe, - TBlobStorageGroupType::Erasure4Plus2Block, TBlobStorageGroupType::Erasure3Plus2Block, - TBlobStorageGroupType::Erasure4Plus2Stripe, TBlobStorageGroupType::Erasure3Plus2Stripe, - TBlobStorageGroupType::ErasureMirror3Plus2}) { - auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(erasure, 3U, 8U, 1U); - const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize(); - TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo); - - for (ui32 i = 0; i < 100; ++i) { - for (ui32 j = 0; j < 10; ++j) { - TLogoBlobID id(i, 1, j, 1, 1000, 1); - BasicCheck(groupInfo, orig, id, blobSubgroupSize); - } - } - } - } - - Y_UNIT_TEST(CheckCorrectBehaviourWithHashOverlow) { - auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(TErasureType::ErasureMirror3, 1U, 5U, 1U); - const ui32 blobSubgroupSize = groupInfo->Type.BlobSubgroupSize(); - TOriginalBlobStorageGroupInfo orig(blobSubgroupSize, *groupInfo); - TLogoBlobID id(4550843067551373890, 2564314201, 2840555155, 59, 0, 2230444); - BasicCheck(groupInfo, orig, id, blobSubgroupSize); - } - - Y_UNIT_TEST(Mirror3dcMapper) { - auto groupInfo = std::make_unique<TBlobStorageGroupInfo>(TBlobStorageGroupType::ErasureMirror3, 3U, 5U, 4U); - - std::unique_ptr<IBlobToDiskMapper> mapper{IBlobToDiskMapper::CreateMirror3dcMapper(&groupInfo->GetTopology())}; - - THashMap<TVDiskID, TVector<ui32>> usageMap; - for (const auto& vdisk : groupInfo->GetVDisks()) { - auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); - usageMap.emplace(vd, TVector<ui32>(9)); - } - - TReallyFastRng32 rng(1); - for (ui32 i = 0; i < 10000; ++i) { - const ui32 hash = rng(); - - TBlobStorageGroupInfo::TOrderNums orderNums; - TBlobStorageGroupInfo::TVDiskIds vdisks; - mapper->PickSubgroup(hash, orderNums); - for (const auto &x : orderNums) { - vdisks.push_back(groupInfo->GetVDiskId(x)); - } - - for (ui32 i = 0; i < vdisks.size(); ++i) { - ++usageMap[vdisks[i]][i]; - } - - for (ui32 i = 0; i < 3; ++i) { - // ensure that cells within a column are in the same ring - UNIT_ASSERT_VALUES_EQUAL(vdisks[i].FailRealm, vdisks[i + 3].FailRealm); - UNIT_ASSERT_VALUES_EQUAL(vdisks[i].FailRealm, vdisks[i + 6].FailRealm); - // and from different fail domains - UNIT_ASSERT(vdisks[i].FailDomain != vdisks[i + 3].FailDomain); - UNIT_ASSERT(vdisks[i].FailDomain != vdisks[i + 6].FailDomain); - UNIT_ASSERT(vdisks[i + 3].FailDomain != vdisks[i + 6].FailDomain); - } - UNIT_ASSERT(vdisks[0].FailRealm != vdisks[1].FailRealm); - UNIT_ASSERT(vdisks[0].FailRealm != vdisks[2].FailRealm); - UNIT_ASSERT(vdisks[1].FailRealm != vdisks[2].FailRealm); - - THashMap<TVDiskID, ui32> disk2index; - for (ui32 i = 0; i < vdisks.size(); ++i) { - disk2index[vdisks[i]] = i; - } - - for (const auto& vdisk : groupInfo->GetVDisks()) { - auto vd = groupInfo->GetVDiskId(vdisk.OrderNumber); - auto it = disk2index.find(vd); - bool isReplicaFor = it != disk2index.end(); - UNIT_ASSERT_VALUES_EQUAL(mapper->GetIdxInSubgroup(vd, hash), isReplicaFor ? it->second : 9); - UNIT_ASSERT_VALUES_EQUAL(mapper->BelongsToSubgroup(vd, hash), isReplicaFor); - } - - for (ui32 i = 0; i < vdisks.size(); ++i) { - UNIT_ASSERT_EQUAL(mapper->GetVDiskInSubgroup(i, hash), vdisks[i]); - } - } - - TVector<double> xv; - for (const auto& kv : usageMap) { - Cerr << kv.first.ToString() << "#"; - for (ui32 i : kv.second) { - xv.push_back(i); - Cerr << " " << i; - } - Cerr << Endl; - } - - double mean = 0; - for (double x : xv) { - mean += x; - } - mean /= xv.size(); - - double dev = 0; - for (double x : xv) { - double d = x - mean; - dev += d * d; - } - dev = sqrt(dev / xv.size()); - - Cerr << "mean# " << mean << " dev# " << dev << Endl; - } - -} diff --git a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp b/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp deleted file mode 100644 index 900f40a0b7b..00000000000 --- a/ydb/core/blobstorage/groupinfo/blobstorage_groupinfo_partlayout_ut.cpp +++ /dev/null @@ -1,160 +0,0 @@ -#include <library/cpp/testing/unittest/registar.h> -#include "blobstorage_groupinfo_partlayout.h" -#include <thread> -#include <mutex> -#include <condition_variable> - -using namespace NKikimr; - -ui32 CountEffectiveReplicas(const TSubgroupPartLayout& layout, ui32 numRows, ui32 numCols) { - // number of effective replicas is calculated as the maximum number of rows that can be mapped to unique columns in - // layout with bit at intersecting cell set to zero; here we state that numCols >= numRows - UNIT_ASSERT(numCols >= numRows); - std::vector<ui32> index(numCols); - for (size_t i = 0; i < index.size(); ++i) { - index[i] = i; - } - ui32 res = 0; - do { - ui32 value = 0; - for (ui32 row = 0; row < numRows; ++row) { - // extract the cell at row, col and count it if it is nonzero - value += layout.GetDisksWithPart(row) >> index[row] & 1; - } - if (value > res) { - res = value; - } - } while (std::next_permutation(index.begin(), index.end())); - return res; -} - -class TCheckQueue { - std::mutex Mutex; - std::list<std::thread> Threads; - std::deque<TSubgroupPartLayout> Queue; - std::condition_variable Cvar, ReverseCvar; - volatile bool Done = false; - std::atomic_size_t CasesChecked = 0; - std::atomic_uint64_t Timing = 0; - -public: - TCheckQueue(TBlobStorageGroupType gtype) { - for (int n = std::thread::hardware_concurrency(); n; --n) { - Threads.emplace_back([=] { - for (;;) { - TSubgroupPartLayout layout; - { - std::unique_lock<std::mutex> lock(Mutex); - Cvar.wait(lock, [&] { return Done || !Queue.empty(); }); - if (Queue.empty() && Done) { - return; - } - Y_VERIFY(!Queue.empty()); - layout = std::move(Queue.front()); - Queue.pop_front(); - if (Queue.size() < 1024) { - ReverseCvar.notify_one(); - } - } - - // count effective replicas - THPTimer timer; - ui32 count = layout.CountEffectiveReplicas(gtype); - Timing += 1e9 * timer.Passed(); - - // compare it with generic value - ui32 generic = CountEffectiveReplicas(layout, gtype.TotalPartCount(), gtype.BlobSubgroupSize()); - - // verify the value - Y_VERIFY(count == generic, "count# %" PRIu32 " generic# %" PRIu32 " layout# %s erasure# %s", - count, generic, layout.ToString(gtype).data(), - TBlobStorageGroupType::ErasureSpeciesName(gtype.GetErasure()).data()); - - ++CasesChecked; - } - }); - } - } - - ~TCheckQueue() { - { - std::unique_lock<std::mutex> lock(Mutex); - Done = true; - } - Cvar.notify_all(); - for (auto& thread : Threads) { - thread.join(); - } - Cerr << "Checked " << (size_t)CasesChecked << " cases, took " << (ui64)Timing / 1000 << " us" << Endl; - } - - void operator ()(const TSubgroupPartLayout& layout) { - std::unique_lock<std::mutex> lock(Mutex); - ReverseCvar.wait(lock, [&] { return Queue.size() < 1024; }); - Queue.push_back(layout); - Cvar.notify_one(); - } -}; - -void TestErasureSet(ui32 firstIdx, ui32 step) { - for (ui32 erasure = firstIdx; erasure < TBlobStorageGroupType::ErasureSpeciesCount; erasure += step) { - if (erasure == TBlobStorageGroupType::ErasureMirror3dc || erasure == TBlobStorageGroupType::ErasureMirror3of4) { - continue; - } - TBlobStorageGroupType gtype(static_cast<TBlobStorageGroupType::EErasureSpecies>(erasure)); - if (gtype.BlobSubgroupSize() > 8) { - continue; - } - Cerr << "testing erasure " << TBlobStorageGroupType::ErasureSpeciesName(erasure) << Endl; - const ui32 totalPartCount = gtype.TotalPartCount(); - const ui32 blobSubgroupSize = gtype.BlobSubgroupSize(); - TCheckQueue checker(gtype); - for (ui32 main = 0; main < 1 << totalPartCount; ++main) { - Cerr << " main# " << main << Endl; - std::vector<ui32> handoffs(gtype.Handoff(), 0); - for (;;) { - // generate subgroup layout - TSubgroupPartLayout layout; - for (ui32 nodeId = 0; nodeId < blobSubgroupSize; ++nodeId) { - if (nodeId < totalPartCount) { - if (main & 1 << nodeId) { - layout.AddItem(nodeId, nodeId, gtype); - } - } else { - for (ui32 i = 0; i < totalPartCount; ++i) { - if (handoffs[nodeId - totalPartCount] & 1 << i) { - layout.AddItem(nodeId, i, gtype); - } - } - } - } - - checker(layout); - - // increment handoffs - ui32 carry = 1; - for (size_t i = 0; carry && i < handoffs.size(); ++i) { - carry = !(++handoffs[i] & (1 << totalPartCount) - 1); - } - if (carry) { - break; - } - } - } - } -} - -Y_UNIT_TEST_SUITE(TSubgroupPartLayoutTest) { - Y_UNIT_TEST(CountEffectiveReplicas1of4) { - TestErasureSet(0, 4); - } - Y_UNIT_TEST(CountEffectiveReplicas2of4) { - TestErasureSet(1, 4); - } - Y_UNIT_TEST(CountEffectiveReplicas3of4) { - TestErasureSet(2, 4); - } - Y_UNIT_TEST(CountEffectiveReplicas4of4) { - TestErasureSet(3, 4); - } -} |