diff options
author | serg-belyakov <[email protected]> | 2023-02-06 17:38:53 +0300 |
---|---|---|
committer | serg-belyakov <[email protected]> | 2023-02-06 17:38:53 +0300 |
commit | 813befe691b16fcf51beec6e725d4e04365dfd55 (patch) | |
tree | 944b82b81f9301ba2c08809cb42ed79073ef32ec | |
parent | 7311c32e97bf999d948b40d90d320089fd350112 (diff) |
Multiple blob depot UT improvements,
Multiple blob depot UT improvements
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot.cpp | 8 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.cpp | 51 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.h | 19 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot_fat.cpp | 10 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot_test_env.h (renamed from ydb/core/blobstorage/ut_blobstorage/blob_depot_auxiliary_structures.h) | 116 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.cpp | 69 | ||||
-rw-r--r-- | ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.h | 91 |
7 files changed, 188 insertions, 176 deletions
diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot.cpp b/ydb/core/blobstorage/ut_blobstorage/blob_depot.cpp index ef6e9c33184..8b3ce400caf 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot.cpp @@ -1,8 +1,6 @@ #include <ydb/core/blobstorage/ut_blobstorage/lib/env.h> #include <ydb/core/blob_depot/events.h> -#include "blob_depot_event_managers.h" -#include "blob_depot_auxiliary_structures.h" #include "blob_depot_test_functions.h" #include <util/random/entropy.h> @@ -128,7 +126,8 @@ Y_UNIT_TEST_SUITE(BlobDepot) { Seed().LoadOrFail(&seed, sizeof(seed)); TBlobDepotTestEnvironment tenv(seed); - TestLoadPutAndGet(tenv, 15, tenv.RegularGroups[0], 100, 1 << 10, 500, true, { 5, 1, 5, 1, 1, 0 }); + TestLoadPutAndGet(tenv, 15, tenv.RegularGroups[0], 100, 1 << 10, 500, true, 10, { 5, 1, 5, 1, 1, 0 }); + // no blob depot restarts performed } Y_UNIT_TEST(DecommitVerifiedRandom) { @@ -136,6 +135,7 @@ Y_UNIT_TEST_SUITE(BlobDepot) { Seed().LoadOrFail(&seed, sizeof(seed)); TBlobDepotTestEnvironment tenv(seed); - TestVerifiedRandom(tenv, 8, 15, tenv.RegularGroups[0], 1000, 499, { 10, 10, 3, 3, 2, 1, 1, 3, 3, 0 }); + TestVerifiedRandom(tenv, 8, 15, tenv.RegularGroups[0], 1000, 500, 10, { 10, 10, 3, 3, 2, 1, 1, 3, 3, 0 }); + // no blob depot restarts performed } } diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.cpp b/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.cpp index 9066372453a..842a9f142be 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.cpp @@ -1,7 +1,7 @@ #include <ydb/core/blobstorage/ut_blobstorage/lib/env.h> #include <ydb/core/blob_depot/events.h> -#include <blob_depot_event_managers.h> +#include "blob_depot_event_managers.h" // #define LOG_PUT // #define LOG_GET @@ -21,6 +21,10 @@ bool IsCollected(const TBlobInfo& blob, ui32 softCollectGen, ui32 softCollectSte } +TInstant MakeDeadline(TEnvironmentSetup& env, bool withDeadline, TDuration deadline = TDuration::Seconds(10)) { + return withDeadline ? env.Runtime->GetClock() +deadline : TInstant::Max(); +} + std::unique_ptr<IEventHandle> CaptureAnyResult(TEnvironmentSetup& env, TActorId sender) { std::set<TActorId> ids{sender}; @@ -63,8 +67,10 @@ void SendTEvPut(TEnvironmentSetup& env, TActorId sender, ui32 groupId, TLogoBlob } TAutoPtr<TEventHandle<TEvBlobStorage::TEvPutResult>> CaptureTEvPutResult(TEnvironmentSetup& env, - TActorId sender, bool termOnCapture) { - auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvPutResult>(sender, termOnCapture); + TActorId sender, bool termOnCapture, bool withDeadline) { + const TInstant deadline = MakeDeadline(env, withDeadline); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvPutResult>(sender, termOnCapture, deadline); + UNIT_ASSERT(res); #ifdef LOG_PUT Cerr << "Response# " << res->Get()->ToString() << Endl; @@ -103,10 +109,10 @@ void VerifyTEvPutResult(TAutoPtr<TEventHandle<TEvBlobStorage::TEvPutResult>> res } } -void VerifiedPut(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, TBlobInfo& blob, TBSState& state) { +void VerifiedPut(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, TBlobInfo& blob, TBSState& state, bool withDeadline) { auto sender = env.Runtime->AllocateEdgeActor(nodeId); SendTEvPut(env, sender, groupId, blob.Id, blob.Data); - auto res = CaptureTEvPutResult(env, sender, true); + auto res = CaptureTEvPutResult(env, sender, true, withDeadline); VerifyTEvPutResult(res.Release(), blob, state); } @@ -125,7 +131,7 @@ void SendTEvGet(TEnvironmentSetup& env, TActorId sender, ui32 groupId, TLogoBlob } TAutoPtr<TEventHandle<TEvBlobStorage::TEvGetResult>> CaptureTEvGetResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture, bool withDeadline) { - const TInstant deadline = withDeadline ? env.Runtime->GetClock() + TDuration::Seconds(10) : TInstant::Max(); + const TInstant deadline = MakeDeadline(env, withDeadline); auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvGetResult>(sender, termOnCapture, deadline); UNIT_ASSERT(res); @@ -212,8 +218,8 @@ void SendTEvGet(TEnvironmentSetup& env, TActorId sender, ui32 groupId, std::vect }); } -TAutoPtr<TEventHandle<TEvBlobStorage::TEvGetResult>> CaptureMultiTEvGetResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture, bool withDeadline) { - const TInstant deadline = withDeadline ? env.Runtime->GetClock() + TDuration::Seconds(10) : TInstant::Max(); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvGetResult>> CaptureMultiTEvGetResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture, bool withDeadline) { + const TInstant deadline = MakeDeadline(env, withDeadline); auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvGetResult>(sender, termOnCapture, deadline); UNIT_ASSERT(res); @@ -289,8 +295,9 @@ void SendTEvRange(TEnvironmentSetup& env, TActorId sender, ui32 groupId, ui64 ta } TAutoPtr<TEventHandle<TEvBlobStorage::TEvRangeResult>> CaptureTEvRangeResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture, bool withDeadline) { - const TInstant deadline = withDeadline ? env.Runtime->GetClock() + TDuration::Seconds(10) : TInstant::Max(); + const TInstant deadline = MakeDeadline(env, withDeadline); auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvRangeResult>(sender, termOnCapture, deadline); + UNIT_ASSERT(res); #ifdef LOG_RANGE Cerr << "Response# " << res->Get()->ToString() << Endl; @@ -384,9 +391,9 @@ void SendTEvDiscover(TEnvironmentSetup& env, TActorId sender, ui32 groupId, ui64 } TAutoPtr<TEventHandle<TEvBlobStorage::TEvDiscoverResult>> CaptureTEvDiscoverResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture, bool withDeadline) { - const TInstant deadline = withDeadline ? env.Runtime->GetClock() + TDuration::Seconds(10) : TInstant::Max(); + const TInstant deadline = MakeDeadline(env, withDeadline); auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvDiscoverResult>(sender, termOnCapture, deadline); - UNIT_ASSERT_C(res, "Timeout - no TEvDiscoverResult received"); + UNIT_ASSERT(res); #ifdef LOG_DISCOVER Cerr << "Response# " << res->Get()->ToString() << Endl; @@ -483,8 +490,11 @@ void SendTEvCollectGarbage(TEnvironmentSetup& env, TActorId sender, ui32 groupId }); } -TAutoPtr<TEventHandle<TEvBlobStorage::TEvCollectGarbageResult>> CaptureTEvCollectGarbageResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture) { - auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvCollectGarbageResult>(sender, termOnCapture); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvCollectGarbageResult>> CaptureTEvCollectGarbageResult(TEnvironmentSetup& env, TActorId sender, + bool termOnCapture, bool withDeadline) { + const TInstant deadline = MakeDeadline(env, withDeadline); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvCollectGarbageResult>(sender, termOnCapture, deadline); + UNIT_ASSERT(res); #ifdef LOG_COLLECT_GARBAGE Cerr << "Response# " << res->Get()->ToString() << Endl; @@ -575,7 +585,7 @@ void VerifiedCollectGarbage(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 recordGeneration, ui32 perGenerationCounter, ui32 channel, bool collect, ui32 collectGeneration, ui32 collectStep, TVector<TLogoBlobID> *keep, TVector<TLogoBlobID> *doNotKeep, - bool isMultiCollectAllowed, bool hard, std::vector<TBlobInfo>& blobs, TBSState& state) + bool isMultiCollectAllowed, bool hard, std::vector<TBlobInfo>& blobs, TBSState& state, bool withDeadline) { auto sender = env.Runtime->AllocateEdgeActor(nodeId); @@ -591,7 +601,7 @@ void VerifiedCollectGarbage(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, SendTEvCollectGarbage(env, sender, groupId, tabletId, recordGeneration, perGenerationCounter, channel, collect, collectGeneration, collectStep, keep, doNotKeep, isMultiCollectAllowed, hard); - auto res = CaptureTEvCollectGarbageResult(env, sender, true); + auto res = CaptureTEvCollectGarbageResult(env, sender, true, withDeadline); VerifyTEvCollectGarbageResult(res.Release(), tabletId, recordGeneration, perGenerationCounter, channel, collect, collectGeneration, collectStep, copyKeep.Get(), copyDoNotKeep.Get(), isMultiCollectAllowed, hard, blobs, state); } @@ -609,8 +619,11 @@ void SendTEvBlock(TEnvironmentSetup& env, TActorId sender, ui32 groupId, ui64 ta }); } -TAutoPtr<TEventHandle<TEvBlobStorage::TEvBlockResult>> CaptureTEvBlockResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture) { - auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvBlockResult>(sender, termOnCapture); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvBlockResult>> CaptureTEvBlockResult(TEnvironmentSetup& env, TActorId sender, + bool termOnCapture, bool withDeadline) { + const TInstant deadline = MakeDeadline(env, withDeadline); + auto res = env.WaitForEdgeActorEvent<TEvBlobStorage::TEvBlockResult>(sender, termOnCapture, deadline); + UNIT_ASSERT(res); #ifdef LOG_BLOCK Cerr << "Response# " << res->Get()->ToString() << Endl; @@ -637,10 +650,10 @@ void VerifyTEvBlockResult(TAutoPtr<TEventHandle<TEvBlobStorage::TEvBlockResult>> } } -void VerifiedBlock(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 generation, TBSState& state) { +void VerifiedBlock(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 generation, TBSState& state, bool withDeadline) { auto sender = env.Runtime->AllocateEdgeActor(nodeId); SendTEvBlock(env, sender, groupId, tabletId, generation); - auto res = CaptureTEvBlockResult(env, sender, true); + auto res = CaptureTEvBlockResult(env, sender, true, withDeadline); VerifyTEvBlockResult(res.Release(), tabletId, generation, state); } diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.h b/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.h index a63f783a95e..0a62ee8f503 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.h +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot_event_managers.h @@ -3,7 +3,7 @@ #include <ydb/core/blobstorage/ut_blobstorage/lib/env.h> #include <ydb/core/blob_depot/events.h> -#include <blob_depot_auxiliary_structures.h> +#include "blob_depot_test_env.h" #include <optional> @@ -11,9 +11,9 @@ std::unique_ptr<IEventHandle> CaptureAnyResult(TEnvironmentSetup& env, TActorId /* --------------------------------- PUT --------------------------------- */ void SendTEvPut(TEnvironmentSetup& env, TActorId sender, ui32 groupId, TLogoBlobID id, TString data, ui64 cookie = 0); -TAutoPtr<TEventHandle<TEvBlobStorage::TEvPutResult>> CaptureTEvPutResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture = true); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvPutResult>> CaptureTEvPutResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture = true, bool withDeadline = true); void VerifyTEvPutResult(TAutoPtr<TEventHandle<TEvBlobStorage::TEvPutResult>> res, TBlobInfo& blob, TBSState& state); -void VerifiedPut(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, TBlobInfo& blob, TBSState& state); +void VerifiedPut(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, TBlobInfo& blob, TBSState& state, bool withDeadline = true); /* --------------------------------- GET --------------------------------- */ void SendTEvGet(TEnvironmentSetup& env, TActorId sender, ui32 groupId, TLogoBlobID id, @@ -49,7 +49,8 @@ void VerifiedRange(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 table /* --------------------------------- DISCOVER --------------------------------- */ void SendTEvDiscover(TEnvironmentSetup& env, TActorId sender, ui32 groupId, ui64 tabletId, ui32 minGeneration, bool readBody, bool discoverBlockedGeneration, ui32 forceBlockedGeneration, bool fromLeader, ui64 cookie = 0); -TAutoPtr<TEventHandle<TEvBlobStorage::TEvDiscoverResult>> CaptureTEvDiscoverResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture = true, bool withDeadline = true); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvDiscoverResult>> CaptureTEvDiscoverResult(TEnvironmentSetup& env, TActorId sender, + bool termOnCapture = true, bool withDeadline = true); void VerifyTEvDiscoverResult(TAutoPtr<TEventHandle<TEvBlobStorage::TEvDiscoverResult>> res, ui64 tabletId, ui32 minGeneration, bool readBody, bool discoverBlockedGeneration, ui32 forceBlockedGeneration, bool fromLeader, std::vector<TBlobInfo>& blobs, TBSState& state); void VerifiedDiscover(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 minGeneration, bool readBody, @@ -61,7 +62,8 @@ void SendTEvCollectGarbage(TEnvironmentSetup& env, TActorId sender, ui32 groupId bool collect, ui32 collectGeneration, ui32 collectStep, TVector<TLogoBlobID> *keep, TVector<TLogoBlobID> *doNotKeep, bool isMultiCollectAllowed, bool hard, ui64 cookie = 0); -TAutoPtr<TEventHandle<TEvBlobStorage::TEvCollectGarbageResult>> CaptureTEvCollectGarbageResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture = true); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvCollectGarbageResult>> CaptureTEvCollectGarbageResult(TEnvironmentSetup& env, TActorId sender, + bool termOnCapture = true, bool withDeadline = true); void VerifyTEvCollectGarbageResult(TAutoPtr<TEventHandle<TEvBlobStorage::TEvCollectGarbageResult>> res, ui64 tabletId, ui32 recordGeneration, ui32 perGenerationCounter, ui32 channel, bool collect, ui32 collectGeneration, @@ -71,10 +73,11 @@ void VerifiedCollectGarbage(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 recordGeneration, ui32 perGenerationCounter, ui32 channel, bool collect, ui32 collectGeneration, ui32 collectStep, TVector<TLogoBlobID> *keep, TVector<TLogoBlobID> *doNotKeep, - bool isMultiCollectAllowed, bool hard, std::vector<TBlobInfo>& blobs, TBSState& state); + bool isMultiCollectAllowed, bool hard, std::vector<TBlobInfo>& blobs, TBSState& state, bool withDeadline = true); /* --------------------------------- BLOCK --------------------------------- */ void SendTEvBlock(TEnvironmentSetup& env, TActorId sender, ui32 groupId, ui64 tabletId, ui32 generation, ui64 cookie = 0); -TAutoPtr<TEventHandle<TEvBlobStorage::TEvBlockResult>> CaptureTEvBlockResult(TEnvironmentSetup& env, TActorId sender, bool termOnCapture = true); +TAutoPtr<TEventHandle<TEvBlobStorage::TEvBlockResult>> CaptureTEvBlockResult(TEnvironmentSetup& env, TActorId sender, + bool termOnCapture = true, bool withDeadline = true); void VerifyTEvBlockResult(TAutoPtr<TEventHandle<TEvBlobStorage::TEvBlockResult>> res, ui64 tabletId, ui32 generation, TBSState& state); -void VerifiedBlock(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 generation, TBSState& state); +void VerifiedBlock(TEnvironmentSetup& env, ui32 nodeId, ui32 groupId, ui64 tabletId, ui32 generation, TBSState& state, bool withDeadline = true); diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot_fat.cpp b/ydb/core/blobstorage/ut_blobstorage/blob_depot_fat.cpp index eac9d1253e0..a57d7858f22 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot_fat.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot_fat.cpp @@ -2,7 +2,7 @@ #include <ydb/core/blob_depot/events.h> #include "blob_depot_event_managers.h" -#include "blob_depot_auxiliary_structures.h" +#include "blob_depot_test_env.h" #include "blob_depot_test_functions.h" #include <util/random/entropy.h> @@ -13,17 +13,17 @@ Y_UNIT_TEST_SUITE(BlobDepotFat) { Y_UNIT_TEST(FatVerifiedRandom) { ui32 seed; Seed().LoadOrFail(&seed, sizeof(seed)); - TBlobDepotTestEnvironment tenv(seed, 1, 8, TBlobStorageGroupType::ErasureMirror3of4, 1500); + TBlobDepotTestEnvironment tenv(seed, 1, 8, TBlobStorageGroupType::ErasureMirror3of4); - TestVerifiedRandom(tenv, 8, 100, tenv.BlobDepot, 1e9); + TestVerifiedRandom(tenv, 8, 100, tenv.BlobDepot, 1e9, 1e9, 1500); } Y_UNIT_TEST(FatDecommitVerifiedRandom) { ui32 seed; Seed().LoadOrFail(&seed, sizeof(seed)); - TBlobDepotTestEnvironment tenv(seed, 1, 8, TBlobStorageGroupType::ErasureMirror3of4, 1500); + TBlobDepotTestEnvironment tenv(seed, 1, 8, TBlobStorageGroupType::ErasureMirror3of4); - TestVerifiedRandom(tenv, 8, 100, tenv.BlobDepot, 1e9, 1000); + TestVerifiedRandom(tenv, 8, 100, tenv.BlobDepot, 1e9, 1000, 1500); } /* ----- Restore is not implemented in blob depot ------ diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot_auxiliary_structures.h b/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_env.h index 48a78160430..76bfb32411f 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot_auxiliary_structures.h +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_env.h @@ -2,6 +2,9 @@ #include <ydb/core/base/logoblob.h> +#include <util/random/mersenne.h> +#include <util/random/random.h> + #include <vector> #include <map> @@ -92,8 +95,6 @@ struct TIntervals { } }; -/* ------------------------ REQUEST ARGUMENTS ----------------------- */ - struct TEvArgs { enum EEventType : ui32 { PUT, @@ -175,3 +176,114 @@ struct TEvRangeArgs : public TEvArgs { bool MustRestoreFirst = false; bool IndexOnly = false; }; + +struct TBlobDepotTestEnvironment { + TMersenne<ui32> Mt; + TMersenne<ui64> Mt64; + + std::unique_ptr<TEnvironmentSetup> Env; + std::vector<ui32> RegularGroups; + ui32 BlobDepot; + ui32 BlobDepotTabletId; + + TBlobDepotTestEnvironment(ui32 seed = 0, ui32 numGroups = 1, ui32 nodeCount = 8, + TBlobStorageGroupType erasure = TBlobStorageGroupType::ErasureMirror3of4) + : Mt(seed) + , Mt64(seed) { + Cerr << "Mersenne random seed " << seed << Endl; + ConfigureEnvironment(numGroups, Env, RegularGroups, BlobDepot, nodeCount, erasure); + BlobDepotTabletId = 0; + } + + void ConfigureEnvironment(ui32 numGroups, std::unique_ptr<TEnvironmentSetup>& envPtr, std::vector<ui32>& regularGroups, ui32& blobDepot, + ui32 nodeCount = 8, TBlobStorageGroupType erasure = TBlobStorageGroupType::ErasureMirror3of4) { + envPtr = std::make_unique<TEnvironmentSetup>(TEnvironmentSetup::TSettings{ + .NodeCount = nodeCount, + .Erasure = erasure, + .SetupHive = true, + }); + + envPtr->CreateBoxAndPool(1, numGroups); + envPtr->Sim(TDuration::Seconds(20)); + + regularGroups = envPtr->GetGroups(); + + NKikimrBlobStorage::TConfigRequest request; + TString virtualPool = "virtual_pool"; + { + auto *cmd = request.AddCommand()->MutableDefineStoragePool(); + cmd->SetBoxId(1); + cmd->SetName(virtualPool); + cmd->SetErasureSpecies("none"); + cmd->SetVDiskKind("Default"); + } + { + auto *cmd = request.AddCommand()->MutableAllocateVirtualGroup(); + cmd->SetName("vg"); + cmd->SetHiveId(envPtr->Runtime->GetAppData()->DomainsInfo->HivesByHiveUid.begin()->second); + cmd->SetStoragePoolName(virtualPool); + auto *prof = cmd->AddChannelProfiles(); + prof->SetStoragePoolName(envPtr->StoragePoolName); + prof->SetCount(2); + prof = cmd->AddChannelProfiles(); + prof->SetStoragePoolName(envPtr->StoragePoolName); + prof->SetChannelKind(NKikimrBlobDepot::TChannelKind::Data); + prof->SetCount(2); + } + + auto response = envPtr->Invoke(request); + UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); + blobDepot = response.GetStatus(1).GetGroupId(0); + + envPtr->Sim(TDuration::Seconds(5)); // some time for blob depot to crank up + } + + TString DataGen(ui32 len) { + TString res = ""; + for (ui32 i = 0; i < len; ++i) { + res += 'A' + Mt.GenRand() % ('z' - 'A'); + } + return res; + } + + ui32 Rand(ui32 a, ui32 b) { + if (a >= b) { + return a; + } + return Mt.GenRand() % (b - a) + a; + } + + ui32 Rand(ui32 b) { + return Rand(0, b); + } + + ui32 Rand() { + return Mt.GenRand(); + } + + ui32 Rand64() { + return Mt64.GenRand(); + } + + template <class T> + T& Rand(std::vector<T>& v) { + return v[Rand(v.size())]; + } + + ui32 SeedRand(ui32 a, ui32 b, ui32 seed) { + TMersenne<ui32> temp(seed); + if (a >= b) { + return a; + } + return temp.GenRand() % (b - a) + a; + } + + ui32 SeedRand(ui32 b, ui32 seed) { + return SeedRand(0, b, seed); + } + + template <class T> + const T& Rand(const std::vector<T>& v) { + return v[Rand(v.size())]; + } +}; diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.cpp b/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.cpp index 034fa7221de..f36043daf8a 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.cpp +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.cpp @@ -1,57 +1,10 @@ #include <ydb/core/blobstorage/ut_blobstorage/lib/env.h> #include <ydb/core/blob_depot/events.h> -#include <util/random/mersenne.h> -#include <util/random/random.h> - #include <algorithm> #include <random> -#include "blob_depot_event_managers.h" #include "blob_depot_test_functions.h" -#include "blob_depot_auxiliary_structures.h" - -void ConfigureEnvironment(ui32 numGroups, std::unique_ptr<TEnvironmentSetup>& envPtr, std::vector<ui32>& regularGroups, ui32& blobDepot, ui32 nodeCount, TBlobStorageGroupType erasure) { - envPtr = std::make_unique<TEnvironmentSetup>(TEnvironmentSetup::TSettings{ - .NodeCount = nodeCount, - .Erasure = erasure, - .SetupHive = true, - }); - - envPtr->CreateBoxAndPool(1, numGroups); - envPtr->Sim(TDuration::Seconds(20)); - - regularGroups = envPtr->GetGroups(); - - NKikimrBlobStorage::TConfigRequest request; - TString virtualPool = "virtual_pool"; - { - auto *cmd = request.AddCommand()->MutableDefineStoragePool(); - cmd->SetBoxId(1); - cmd->SetName(virtualPool); - cmd->SetErasureSpecies("none"); - cmd->SetVDiskKind("Default"); - } - { - auto *cmd = request.AddCommand()->MutableAllocateVirtualGroup(); - cmd->SetName("vg"); - cmd->SetHiveId(envPtr->Runtime->GetAppData()->DomainsInfo->HivesByHiveUid.begin()->second); - cmd->SetStoragePoolName(virtualPool); - auto *prof = cmd->AddChannelProfiles(); - prof->SetStoragePoolName(envPtr->StoragePoolName); - prof->SetCount(2); - prof = cmd->AddChannelProfiles(); - prof->SetStoragePoolName(envPtr->StoragePoolName); - prof->SetChannelKind(NKikimrBlobDepot::TChannelKind::Data); - prof->SetCount(2); - } - - auto response = envPtr->Invoke(request); - UNIT_ASSERT_C(response.GetSuccess(), response.GetErrorDescription()); - blobDepot = response.GetStatus(1).GetGroupId(0); - - envPtr->Sim(TDuration::Seconds(5)); // some time for blob depot to crank up -} void DecommitGroup(TBlobDepotTestEnvironment& tenv, ui32 groupId) { TString blobDepotPool = "decommit_blob_depot_pool"; @@ -494,7 +447,8 @@ void TestRestoreRange(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 group } } -void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 tabletId0, ui32 groupId, ui32 iterationsNum, ui32 decommitStep, std::vector<ui32> probabilities) { +void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 tabletId0, ui32 groupId, + ui32 iterationsNum, ui32 decommitStep, ui32 timeLimitSec, std::vector<ui32> probabilities) { enum EActions { ALTER = 0, PUT, @@ -507,6 +461,11 @@ void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 ta COLLECT_GARBAGE_SOFT, RESTART_BLOB_DEPOT, }; + + std::vector<std::string> actionName = { + "ALTER", "PUT", "GET", "MULTIGET", "RANGE", "BLOCK", "DISCOVER", "COLLECT_GARBAGE_HARD", "COLLECT_GARBAGE_SOFT", "RESTART_BLOB_DEPOT" + }; + std::vector<ui32> probs = probabilities; TIntervals act(probs); @@ -526,12 +485,15 @@ void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 ta ui32 perGenCtr = 0; + THPTimer timer; + for (ui32 iteration = 0; iteration < iterationsNum; ++iteration) { if (iteration == decommitStep) { DecommitGroup(tenv, groupId); continue; } - if (tenv.IsFinished()) { + + if (timeLimitSec && timer.Passed() > timeLimitSec) { break; } ui32 tablet = tenv.Rand(tablets.size()); @@ -547,7 +509,7 @@ void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 ta ui32 hardCollectStep = state[tabletId].Channels[channel].HardCollectStep; ui32 action = act.GetInterval(tenv.Rand(act.UpperLimit())); - // Cerr << action << Endl; + // Cerr << "iteration# " << iteration << " action# " << actionName[action] << " timer# " << timer.Passed() << Endl; switch (action) { case EActions::ALTER: { @@ -732,7 +694,8 @@ void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 ta } } -void TestLoadPutAndGet(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId, ui32 blobsNum, ui32 maxBlobSize, ui32 readsNum, bool decommit, std::vector<ui32> probablities) { +void TestLoadPutAndGet(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId, ui32 blobsNum, ui32 maxBlobSize, + ui32 readsNum, bool decommit, ui32 timeLimitSec, std::vector<ui32> probablities) { enum EActions { GET, MULTIGET, @@ -782,12 +745,14 @@ void TestLoadPutAndGet(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 grou DecommitGroup(tenv, groupId); } + THPTimer timer; + for (ui32 iteration = 0; iteration < readsNum; ++iteration) { ui32 action = act.GetInterval(tenv.Rand(act.UpperLimit())); if (iteration == readsNum - 1) { // Catch all results on the last iteration action = EActions::CATCH_ALL; } - if (tenv.IsFinished()) { + if (timeLimitSec && timer.Passed() > timeLimitSec) { break; } diff --git a/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.h b/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.h index 5ff4f4593e4..822e43684c6 100644 --- a/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.h +++ b/ydb/core/blobstorage/ut_blobstorage/blob_depot_test_functions.h @@ -9,93 +9,10 @@ #include <algorithm> #include <random> -#include <blob_depot_event_managers.h> -#include <blob_depot_auxiliary_structures.h> - -void ConfigureEnvironment(ui32 numGroups, std::unique_ptr<TEnvironmentSetup>& envPtr, std::vector<ui32>& regularGroups, ui32& blobDepot, ui32 nodeCount = 8, - TBlobStorageGroupType erasure = TBlobStorageGroupType::ErasureMirror3of4); - - -struct TBlobDepotTestEnvironment { - TMersenne<ui32> Mt; - TMersenne<ui64> Mt64; - - std::unique_ptr<TEnvironmentSetup> Env; - std::vector<ui32> RegularGroups; - ui32 BlobDepot; - ui32 BlobDepotTabletId; - THPTimer Timer; - ui32 TimeLimit; - TBlobDepotTestEnvironment(ui32 seed = 0, ui32 numGroups = 1, ui32 nodeCount = 8, TBlobStorageGroupType erasure = TBlobStorageGroupType::ErasureMirror3of4, ui32 timeLimit = 0) - : Mt(seed) - , Mt64(seed) - , TimeLimit(timeLimit) { - Cerr << "Mersenne random seed " << seed << Endl; - ConfigureEnvironment(numGroups, Env, RegularGroups, BlobDepot, nodeCount, erasure); - BlobDepotTabletId = 0; - } - - TString DataGen(ui32 len) { - TString res = ""; - for (ui32 i = 0; i < len; ++i) { - res += 'A' + Mt.GenRand() % ('z' - 'A'); - } - return res; - } - - ui32 Rand(ui32 a, ui32 b) { - if (a >= b) { - return a; - } - return Mt.GenRand() % (b - a) + a; - } - - ui32 Rand(ui32 b) { - return Rand(0, b); - } - - ui32 Rand() { - return Mt.GenRand(); - } - - ui32 Rand64() { - return Mt64.GenRand(); - } - - template <class T> - T& Rand(std::vector<T>& v) { - return v[Rand(v.size())]; - } - - ui32 SeedRand(ui32 a, ui32 b, ui32 seed) { - TMersenne<ui32> temp(seed); - if (a >= b) { - return a; - } - return temp.GenRand() % (b - a) + a; - } - - ui32 SeedRand(ui32 b, ui32 seed) { - return SeedRand(0, b, seed); - } - - template <class T> - const T& Rand(const std::vector<T>& v) { - return v[Rand(v.size())]; - } - - bool IsFinished() { - return TimeLimit && Timer.Passed() > TimeLimit; - } -}; - -void DecommitGroup(TBlobDepotTestEnvironment& tenv, ui32 groupId); +#include "blob_depot_event_managers.h" void TestBasicPutAndGet(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId); -TLogoBlobID MinBlobID(ui64 tablet); -TLogoBlobID MaxBlobID(ui64 tablet); - void TestBasicRange(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId); void TestBasicDiscover(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId); @@ -110,6 +27,8 @@ void TestRestoreDiscover(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 gr void TestRestoreRange(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId, ui32 blobsNum, std::vector<TActorId>* vdisks); -void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 tabletId0, ui32 groupId, ui32 iterationsNum, ui32 decommitStep = 1e9, std::vector<ui32> probabilities = { 10, 10, 3, 3, 2, 1, 1, 3, 3, 1 }); +void TestVerifiedRandom(TBlobDepotTestEnvironment& tenv, ui32 nodeCount, ui64 tabletId0, ui32 groupId, ui32 iterationsNum, + ui32 decommitStep = 1e9, ui32 timeLimitSec = 0, std::vector<ui32> probabilities = { 10, 10, 3, 3, 2, 1, 1, 3, 3, 1 }); -void TestLoadPutAndGet(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId, ui32 blobsNum, ui32 maxBlobSize, ui32 readsNum, bool decommit = false, std::vector<ui32> probabilities = { 5, 1, 5, 5, 1, 1 });
\ No newline at end of file +void TestLoadPutAndGet(TBlobDepotTestEnvironment& tenv, ui64 tabletId, ui32 groupId, ui32 blobsNum, ui32 maxBlobSize, ui32 readsNum, + bool decommit = false, ui32 timeLimitSec = 0, std::vector<ui32> probabilities = { 5, 1, 5, 5, 1, 1 });
\ No newline at end of file |