aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Rutkovsky <alexvru@ydb.tech>2024-11-20 11:50:38 +0300
committerGitHub <noreply@github.com>2024-11-20 11:50:38 +0300
commit5876bb106ea873c752ca2814604f5aa3da860222 (patch)
tree6f936d3b30da9740e17802130e93777d851bcd8a
parent8357e1ce980525089d6e1d21d8cc4c2505d69a34 (diff)
downloadydb-5876bb106ea873c752ca2814604f5aa3da860222.tar.gz
Support arbitrary chain set in huge blob keeper heap (#11566)
-rw-r--r--ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_impl.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_request.cpp5
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_state.cpp4
-rw-r--r--ydb/core/blobstorage/dsproxy/group_sessions.cpp4
-rw-r--r--ydb/core/blobstorage/dsproxy/group_sessions.h2
-rw-r--r--ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp10
-rw-r--r--ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp3
-rw-r--r--ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/balance/defs.h6
-rw-r--r--ydb/core/blobstorage/vdisk/balance/sender.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_config.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_config.h1
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp18
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h6
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_events.h8
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp8
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp26
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h12
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp5
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp609
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h137
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp1
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp114
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp12
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h5
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp20
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h4
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp26
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h8
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h4
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h8
-rw-r--r--ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp10
-rw-r--r--ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h8
-rw-r--r--ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h4
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp36
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp8
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h4
-rw-r--r--ydb/core/protos/blobstorage.proto2
44 files changed, 521 insertions, 637 deletions
diff --git a/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h b/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h
index 66805e3a1d..9518ad7ced 100644
--- a/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h
+++ b/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h
@@ -150,7 +150,7 @@ public:
settings->SetWriteSpeedBps(60e6);
settings->SetReadBlockSize(4096);
settings->SetWriteBlockSize(4096);
- settings->SetMinREALHugeBlobInBytes(512 << 10);
+ settings->SetMinHugeBlobInBytes(512 << 10);
}
STRICT_STFUNC(StateFunc, {
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy.h b/ydb/core/blobstorage/dsproxy/dsproxy.h
index d8f06ae9f0..e0ae85c22d 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy.h
+++ b/ydb/core/blobstorage/dsproxy/dsproxy.h
@@ -30,7 +30,7 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER);
constexpr ui32 TypicalPartsInBlob = 6;
constexpr ui32 TypicalDisksInSubring = 8;
-constexpr ui32 MaxBatchedPutSize = 64 * 1024 - 512 - 5; // (MinREALHugeBlobInBytes - 1 - TDiskBlob::HugeBlobOverhead) for ssd and nvme
+constexpr ui32 MaxBatchedPutSize = 64 * 1024 - 512 - 5; // (MinHugeBlobInBytes - 1 - TDiskBlob::HugeBlobOverhead) for ssd and nvme
const TDuration ProxyConfigurationTimeout = TDuration::Seconds(20);
const ui32 ProxyRetryConfigurationInitialTimeout = 200;
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_impl.h b/ydb/core/blobstorage/dsproxy/dsproxy_impl.h
index dfdbe32b70..3d7fb25129 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_impl.h
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_impl.h
@@ -71,7 +71,7 @@ class TBlobStorageGroupProxy : public TActorBootstrapped<TBlobStorageGroupProxy>
bool UseActorSystemTimeInBSQueue;
bool IsLimitedKeyless = false;
bool IsFullMonitoring = false; // current state of monitoring
- ui32 MinREALHugeBlobInBytes = 0;
+ ui32 MinHugeBlobInBytes = 0;
TActorId MonActor;
TIntrusivePtr<TBlobStorageGroupProxyMon> Mon;
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
index 4575b666b8..4ffe2a10e2 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
@@ -178,13 +178,12 @@ namespace NKikimr {
Send(MonActor, new TEvThroughputAddRequest(ev->Get()->HandleClass, bytes));
EnableWilsonTracing(ev, Mon->PutSamplePPM);
- Y_DEBUG_ABORT_UNLESS(MinREALHugeBlobInBytes);
+ Y_DEBUG_ABORT_UNLESS(MinHugeBlobInBytes);
const ui32 partSize = Info->Type.PartSize(ev->Get()->Id);
TInstant now = TActivationContext::Now();
- if (Controls.EnablePutBatching.Update(now) && partSize < MinREALHugeBlobInBytes &&
- partSize <= MaxBatchedPutSize) {
+ if (Controls.EnablePutBatching.Update(now) && partSize < MinHugeBlobInBytes && partSize <= MaxBatchedPutSize) {
NKikimrBlobStorage::EPutHandleClass handleClass = ev->Get()->HandleClass;
TEvBlobStorage::TEvPut::ETactic tactic = ev->Get()->Tactic;
Y_ABORT_UNLESS((ui64)handleClass <= PutHandleClassCount);
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp
index 3cb1b3fd44..b69f3f6aea 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp
@@ -208,7 +208,7 @@ namespace NKikimr {
Y_ABORT_UNLESS(Topology);
Sessions->QueueConnectUpdate(Topology->GetOrderNumber(msg->VDiskId), msg->QueueId, msg->IsConnected,
msg->ExtraBlockChecksSupport, msg->CostModel, *Topology);
- MinREALHugeBlobInBytes = Sessions->GetMinREALHugeBlobInBytes();
+ MinHugeBlobInBytes = Sessions->GetMinHugeBlobInBytes();
if (msg->IsConnected && (CurrentStateFunc() == &TThis::StateEstablishingSessions ||
CurrentStateFunc() == &TThis::StateEstablishingSessionsTimeout)) {
SwitchToWorkWhenGoodToGo();
@@ -216,7 +216,7 @@ namespace NKikimr {
SetStateEstablishingSessions();
}
- Y_DEBUG_ABORT_UNLESS(CurrentStateFunc() != &TThis::StateWork || MinREALHugeBlobInBytes);
+ Y_DEBUG_ABORT_UNLESS(CurrentStateFunc() != &TThis::StateWork || MinHugeBlobInBytes);
if (const ui32 prev = std::exchange(NumUnconnectedDisks, Sessions->GetNumUnconnectedDisks()); prev != NumUnconnectedDisks) {
NodeMon->IncNumUnconnected(NumUnconnectedDisks);
diff --git a/ydb/core/blobstorage/dsproxy/group_sessions.cpp b/ydb/core/blobstorage/dsproxy/group_sessions.cpp
index 8569a94ad9..24b33a06d5 100644
--- a/ydb/core/blobstorage/dsproxy/group_sessions.cpp
+++ b/ydb/core/blobstorage/dsproxy/group_sessions.cpp
@@ -188,8 +188,8 @@ void TGroupSessions::QueueConnectUpdate(ui32 orderNumber, NKikimrBlobStorage::EV
}
}
-ui32 TGroupSessions::GetMinREALHugeBlobInBytes() const {
- return GroupQueues->CostModel ? GroupQueues->CostModel->MinREALHugeBlobInBytes : 0;
+ui32 TGroupSessions::GetMinHugeBlobInBytes() const {
+ return GroupQueues->CostModel ? GroupQueues->CostModel->MinHugeBlobInBytes : 0;
}
ui32 TGroupSessions::GetNumUnconnectedDisks() {
diff --git a/ydb/core/blobstorage/dsproxy/group_sessions.h b/ydb/core/blobstorage/dsproxy/group_sessions.h
index 174364976a..8bb706d4cd 100644
--- a/ydb/core/blobstorage/dsproxy/group_sessions.h
+++ b/ydb/core/blobstorage/dsproxy/group_sessions.h
@@ -210,7 +210,7 @@ namespace NKikimr {
void QueueConnectUpdate(ui32 orderNumber, NKikimrBlobStorage::EVDiskQueueId queueId, bool connected,
bool extraBlockChecksSupport, std::shared_ptr<const TCostModel> costModel, const TBlobStorageGroupInfo::TTopology& topology);
ui32 GetNumUnconnectedDisks();
- ui32 GetMinREALHugeBlobInBytes() const;
+ ui32 GetMinHugeBlobInBytes() const;
};
struct TEvRequestProxySessionsState : TEventLocal<TEvRequestProxySessionsState, TEvBlobStorage::EvRequestProxySessionsState>
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp
index ba0477182a..1df9dc3a81 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp
@@ -394,7 +394,7 @@ class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> {
bool Started = false;
// how many deadline statuses we got
ui64 RequestDeadlines = 0;
- ui32 MinREALHugeBlobInBytes = 0;
+ ui32 MinHugeBlobInBytes = 0;
ui64 LastBatchSize = 0;
@@ -413,8 +413,8 @@ class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> {
void Handle(TEvProxyQueueState::TPtr& ev, const TActorContext& ctx) {
if (ev->Get()->IsConnected && !Started) {
// put logo blob
- MinREALHugeBlobInBytes = ev->Get()->CostModel->MinREALHugeBlobInBytes;
- Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
+ MinHugeBlobInBytes = ev->Get()->CostModel->MinHugeBlobInBytes;
+ Y_ABORT_UNLESS(MinHugeBlobInBytes);
SendPut(ctx);
Started = true;
}
@@ -485,12 +485,12 @@ class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> {
Y_ABORT_UNLESS(status == NKikimrProto::OK || noTimeout && status == NKikimrProto::DEADLINE,
"Event# %s", ev->Get()->ToString().data());
- Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinHugeBlobInBytes);
switch (status) {
case NKikimrProto::OK:
for (auto &item : record.GetItems()) {
- Y_ABORT_UNLESS(item.GetStatus() == (MsgData.size() < MinREALHugeBlobInBytes ? NKikimrProto::OK : NKikimrProto::ERROR));
+ Y_ABORT_UNLESS(item.GetStatus() == (MsgData.size() < MinHugeBlobInBytes ? NKikimrProto::OK : NKikimrProto::ERROR));
}
break;
case NKikimrProto::DEADLINE:
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
index ad9b486e67..5fabd73846 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
@@ -137,7 +137,6 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
bool InitHugeBlobKeeper(const TActorContext &ctx, const TStartingPoints &startingPoints) {
Y_UNUSED(ctx);
- const ui32 oldMinHugeBlobInBytes = 64 << 10;
const ui32 milestoneHugeBlobInBytes = 64 << 10;
const ui32 maxBlobInBytes = 128 << 10;
auto logFunc = [] (const TString) { /* empty */ };
@@ -150,7 +149,6 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
HmCtx->PDiskCtx->Dsk->ChunkSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
- oldMinHugeBlobInBytes,
milestoneHugeBlobInBytes,
maxBlobInBytes,
HmCtx->Config->HugeBlobOverhead,
@@ -169,7 +167,6 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
HmCtx->PDiskCtx->Dsk->ChunkSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
- oldMinHugeBlobInBytes,
milestoneHugeBlobInBytes,
maxBlobInBytes,
HmCtx->Config->HugeBlobOverhead,
diff --git a/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp b/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp
index 0a7b8abd73..b0d3585439 100644
--- a/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp
+++ b/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp
@@ -142,7 +142,7 @@ namespace NBalancing {
const auto& key = It.GetCurKey().LogoBlobID();
- if (Ctx->Cfg.BalanceOnlyHugeBlobs && !Ctx->HugeBlobCtx->IsHugeBlob(GInfo->Type, key, Ctx->MinREALHugeBlobInBytes)) {
+ if (Ctx->Cfg.BalanceOnlyHugeBlobs && !Ctx->HugeBlobCtx->IsHugeBlob(GInfo->Type, key, Ctx->MinHugeBlobInBytes)) {
// skip non huge blobs
continue;
}
diff --git a/ydb/core/blobstorage/vdisk/balance/defs.h b/ydb/core/blobstorage/vdisk/balance/defs.h
index b94d6a3b2f..63f4fecaa9 100644
--- a/ydb/core/blobstorage/vdisk/balance/defs.h
+++ b/ydb/core/blobstorage/vdisk/balance/defs.h
@@ -42,7 +42,7 @@ namespace NKikimr {
TIntrusivePtr<TVDiskConfig> VDiskCfg;
TIntrusivePtr<TBlobStorageGroupInfo> GInfo;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
TBalancingCtx(
const TBalancingCfg& cfg,
@@ -53,7 +53,7 @@ namespace NKikimr {
NKikimr::THullDsSnap snap,
TIntrusivePtr<TVDiskConfig> vDiskCfg,
TIntrusivePtr<TBlobStorageGroupInfo> gInfo,
- ui32 minREALHugeBlobInBytes
+ ui32 minHugeBlobInBytes
)
: Cfg(cfg)
, VCtx(std::move(vCtx))
@@ -64,7 +64,7 @@ namespace NKikimr {
, Snap(std::move(snap))
, VDiskCfg(std::move(vDiskCfg))
, GInfo(std::move(gInfo))
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
{
}
};
diff --git a/ydb/core/blobstorage/vdisk/balance/sender.cpp b/ydb/core/blobstorage/vdisk/balance/sender.cpp
index 0fc8b9083f..a455abb572 100644
--- a/ydb/core/blobstorage/vdisk/balance/sender.cpp
+++ b/ydb/core/blobstorage/vdisk/balance/sender.cpp
@@ -153,7 +153,7 @@ namespace {
auto& data = part.PartsData[i];
auto vDiskId = GetMainReplicaVDiskId(*GInfo, key);
- if (Ctx->HugeBlobCtx->IsHugeBlob(GInfo->GetTopology().GType, part.Key, Ctx->MinREALHugeBlobInBytes)) {
+ if (Ctx->HugeBlobCtx->IsHugeBlob(GInfo->GetTopology().GType, part.Key, Ctx->MinHugeBlobInBytes)) {
auto ev = std::make_unique<TEvBlobStorage::TEvVPut>(
key, data, vDiskId,
true, nullptr,
diff --git a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h
index 4b5c7283ab..3cdb566158 100644
--- a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h
+++ b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h
@@ -93,7 +93,7 @@ public:
DeviceWriteSpeedBps = costModel.WriteSpeedBps;
DeviceReadBlockSize = costModel.ReadBlockSize;
DeviceWriteBlockSize = costModel.WriteBlockSize;
- HugeBlobSize = costModel.MinREALHugeBlobInBytes;
+ HugeBlobSize = costModel.MinHugeBlobInBytes;
}
protected:
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
index 540edfaac5..edfcd7c452 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
@@ -136,9 +136,7 @@ namespace NKikimr {
MinHugeBlobInBytes = 512u << 10u;
break;
}
- OldMinHugeBlobInBytes = MinHugeBlobInBytes; // preserved to migrate entry point state correctly
MilestoneHugeBlobInBytes = 512u << 10u; // for compatibility reasons it must be 512KB
-
}
void TVDiskConfig::Merge(const NKikimrBlobStorage::TVDiskConfig &update) {
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
index dd80ff86ce..70bfdd1ee1 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
@@ -120,7 +120,6 @@ namespace NKikimr {
ui32 HullSstSizeInChunksLevel;
ui32 HugeBlobsFreeChunkReservation;
ui32 MinHugeBlobInBytes;
- ui32 OldMinHugeBlobInBytes;
ui32 MilestoneHugeBlobInBytes;
ui32 HugeBlobOverhead;
ui32 HullCompLevel0MaxSstsAtOnce;
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp
index 9d78d90e15..4d4ed7d0ed 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp
@@ -95,13 +95,13 @@ namespace NKikimr {
}
TCostModel::TCostModel(ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize,
- ui64 writeBlockSize, ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType)
+ ui64 writeBlockSize, ui32 minHugeBlobInBytes, TBlobStorageGroupType gType)
: SeekTimeUs(seekTimeUs)
, ReadSpeedBps(readSpeedBps)
, WriteSpeedBps(writeSpeedBps)
, ReadBlockSize(readBlockSize)
, WriteBlockSize(writeBlockSize)
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
, GType(gType)
{}
@@ -111,7 +111,7 @@ namespace NKikimr {
, WriteSpeedBps(settings.GetWriteSpeedBps())
, ReadBlockSize(settings.GetReadBlockSize())
, WriteBlockSize(settings.GetWriteBlockSize())
- , MinREALHugeBlobInBytes(settings.GetMinREALHugeBlobInBytes())
+ , MinHugeBlobInBytes(settings.GetMinHugeBlobInBytes())
, GType(gType)
{}
@@ -121,7 +121,7 @@ namespace NKikimr {
settings.SetWriteSpeedBps(WriteSpeedBps);
settings.SetReadBlockSize(ReadBlockSize);
settings.SetWriteBlockSize(WriteBlockSize);
- settings.SetMinREALHugeBlobInBytes(MinREALHugeBlobInBytes);
+ settings.SetMinHugeBlobInBytes(MinHugeBlobInBytes);
}
/// READS
@@ -181,7 +181,7 @@ namespace NKikimr {
const NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass();
const ui64 bufSize = record.HasBuffer() ? record.GetBuffer().size() : ev.GetPayload(0).GetSize();
- NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, handleClass, bufSize, true);
+ NPriPut::EHandleType handleType = NPriPut::HandleType(MinHugeBlobInBytes, handleClass, bufSize, true);
if (handleType == NPriPut::Log) {
*logPutInternalQueue = true;
return SmallWriteCost(bufSize);
@@ -198,7 +198,7 @@ namespace NKikimr {
ui64 cost = 0;
for (ui64 idx = 0; idx < record.ItemsSize(); ++idx) {
const ui64 size = ev.GetBufferBytes(idx);
- NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, handleClass, size, true);
+ NPriPut::EHandleType handleType = NPriPut::HandleType(MinHugeBlobInBytes, handleClass, size, true);
if (handleType == NPriPut::Log) {
cost += SmallWriteCost(size);
} else {
@@ -265,7 +265,7 @@ namespace NKikimr {
cost += MovedPatchCostBySize(essence.MovedPatchBlobSize);
}
for (ui64 size : essence.PutBufferSizes) {
- NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, essence.HandleClass, size, true);
+ NPriPut::EHandleType handleType = NPriPut::HandleType(MinHugeBlobInBytes, essence.HandleClass, size, true);
if (handleType == NPriPut::Log) {
cost += SmallWriteCost(size);
} else {
@@ -283,7 +283,7 @@ namespace NKikimr {
str << " WriteSpeedBps# " << WriteSpeedBps;
str << " ReadBlockSize# " << ReadBlockSize;
str << " WriteBlockSize# " << WriteBlockSize;
- str << " MinREALHugeBlobInBytes# " << MinREALHugeBlobInBytes;
+ str << " MinHugeBlobInBytes# " << MinHugeBlobInBytes;
str << " GType# " << GType.ToString();
str << "}";
return str.Str();
@@ -295,7 +295,7 @@ namespace NKikimr {
WriteSpeedBps = std::min(WriteSpeedBps, other.WriteSpeedBps);
ReadBlockSize = std::min(ReadBlockSize, other.ReadBlockSize);
WriteBlockSize = std::min(WriteBlockSize, other.WriteBlockSize);
- MinREALHugeBlobInBytes = std::max(MinREALHugeBlobInBytes, other.MinREALHugeBlobInBytes);
+ MinHugeBlobInBytes = std::max(MinHugeBlobInBytes, other.MinHugeBlobInBytes);
}
// PDisk messages cost
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h
index 16ae661627..51887b8ae9 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h
@@ -58,11 +58,11 @@ namespace NKikimr {
ui64 WriteSpeedBps;
ui64 ReadBlockSize;
ui64 WriteBlockSize;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
TBlobStorageGroupType GType;
TCostModel(ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize, ui64 writeBlockSize,
- ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType);
+ ui32 minHugeBlobInBytes, TBlobStorageGroupType gType);
TCostModel(const NKikimrBlobStorage::TVDiskCostSettings &settings, TBlobStorageGroupType gType);
/// SETTINGS
@@ -95,7 +95,7 @@ namespace NKikimr {
WriteSpeedBps != other.WriteSpeedBps ||
ReadBlockSize != other.ReadBlockSize ||
WriteBlockSize != other.WriteBlockSize ||
- MinREALHugeBlobInBytes != other.MinREALHugeBlobInBytes;
+ MinHugeBlobInBytes != other.MinHugeBlobInBytes;
}
// PDisk messages cost
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_events.h
index b99db787e0..05d89c5080 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_events.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_events.h
@@ -703,8 +703,8 @@ namespace NKikimr {
if (costSettings.HasWriteBlockSize()) {
str << " WriteBlockSize# " << costSettings.GetWriteBlockSize();
}
- if (costSettings.HasMinREALHugeBlobInBytes()) {
- str << " MinREALHugeBlobInBytes# " << costSettings.GetMinREALHugeBlobInBytes();
+ if (costSettings.HasMinHugeBlobInBytes()) {
+ str << " MinHugeBlobInBytes# " << costSettings.GetMinHugeBlobInBytes();
}
str << "}";
}
@@ -3199,9 +3199,9 @@ namespace NKikimr {
////////////////////////////////////////////////////////////////////////////
class TEvMinHugeBlobSizeUpdate : public TEventLocal<TEvMinHugeBlobSizeUpdate, TEvBlobStorage::EvMinHugeBlobSizeUpdate> {
public:
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
- TEvMinHugeBlobSizeUpdate(ui32 minREALHugeBlobInBytes) : MinREALHugeBlobInBytes(minREALHugeBlobInBytes) {
+ TEvMinHugeBlobSizeUpdate(ui32 minHugeBlobInBytes) : MinHugeBlobInBytes(minHugeBlobInBytes) {
};
};
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp
index 25c64e1d73..d74fcafb7d 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp
@@ -8,18 +8,18 @@ namespace NKikimr {
///////////////////////////////////////////////////////////////////////////////////
namespace NPriPut {
- EHandleType HandleType(const ui32 minREALHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
+ EHandleType HandleType(const ui32 minHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
ui32 originalBufSizeWithoutOverhead, bool addHeader) {
// what size of huge blob it would be, if it huge
const ui64 hugeBlobSize = (addHeader ? TDiskBlob::HeaderSize : 0) + originalBufSizeWithoutOverhead;
switch (handleClass) {
case NKikimrBlobStorage::TabletLog:
- return (hugeBlobSize >= minREALHugeBlobSize ? HugeForeground : Log);
+ return (hugeBlobSize >= minHugeBlobSize ? HugeForeground : Log);
case NKikimrBlobStorage::AsyncBlob:
- return (hugeBlobSize >= minREALHugeBlobSize ? HugeBackground : Log);
+ return (hugeBlobSize >= minHugeBlobSize ? HugeBackground : Log);
case NKikimrBlobStorage::UserData:
- return (hugeBlobSize >= minREALHugeBlobSize ? HugeForeground : Log);
+ return (hugeBlobSize >= minHugeBlobSize ? HugeForeground : Log);
default:
Y_ABORT("Unexpected case");
}
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h
index eb86e0b36f..73706918ed 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h
@@ -17,7 +17,7 @@ namespace NKikimr {
HugeBackground = 2 // huge blog, write it with low priority
};
- EHandleType HandleType(const ui32 minREALHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
+ EHandleType HandleType(const ui32 minHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
ui32 originalBufSizeWithoutOverhead, bool addHeader);
} // NPriPut
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
index f8d4b2fec4..2c581f9fbf 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
@@ -3,23 +3,23 @@
namespace NKikimr {
- THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable)
+ THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo,
+ TSearchTable &&searchTable)
: AppendBlockSize(appendBlockSize)
+ , MinHugeBlobInBlocks(minHugeBlobInBlocks)
, AllSlotsInfo(std::move(slotsInfo))
, SearchTable(std::move(searchTable))
{}
const THugeSlotsMap::TSlotInfo *THugeSlotsMap::GetSlotInfo(ui32 size) const {
- ui32 sizeInBlocks = size / AppendBlockSize;
- sizeInBlocks += !(sizeInBlocks * AppendBlockSize == size);
- const ui64 idx = SearchTable.at(sizeInBlocks);
+ const ui32 sizeInBlocks = (size + AppendBlockSize - 1) / AppendBlockSize;
+ Y_ABORT_UNLESS(MinHugeBlobInBlocks <= sizeInBlocks);
+ const ui64 idx = SearchTable.at(sizeInBlocks - MinHugeBlobInBlocks);
return &AllSlotsInfo.at(idx);
}
ui32 THugeSlotsMap::AlignByBlockSize(ui32 size) const {
- ui32 sizeInBlocks = size / AppendBlockSize;
- Y_ABORT_UNLESS(sizeInBlocks, "Blob size to align is smaller than a single block. BlobSize# %" PRIu32, size);
- return sizeInBlocks * AppendBlockSize;
+ return Max(MinHugeBlobInBlocks * AppendBlockSize, size - size % AppendBlockSize);
}
void THugeSlotsMap::Output(IOutputStream &str) const {
@@ -31,11 +31,7 @@ namespace NKikimr {
str << "]}\n";
str << "{SearchTable# [";
for (const auto &idx : SearchTable) {
- if (idx != NoOpIdx) {
- AllSlotsInfo.at(idx).Output(str);
- } else {
- str << "null";
- }
+ AllSlotsInfo.at(idx).Output(str);
str << "\n";
}
str << "]}";
@@ -48,9 +44,9 @@ namespace NKikimr {
}
// check whether this blob is huge one; userPartSize doesn't include any metadata stored along with blob
- bool THugeBlobCtx::IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minREALHugeBlobInBytes) const {
- return gtype.MaxPartSize(fullId) + (AddHeader ? TDiskBlob::HeaderSize : 0) >= minREALHugeBlobInBytes;
- }
+ bool THugeBlobCtx::IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minHugeBlobInBytes) const {
+ return gtype.MaxPartSize(fullId) + (AddHeader ? TDiskBlob::HeaderSize : 0) >= minHugeBlobInBytes;
+ }
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
index 5a26e1e16f..1d7e61f04e 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
@@ -38,16 +38,13 @@ namespace NKikimr {
};
// All slot types
- using TAllSlotsInfo = TVector<TSlotInfo>;
+ using TAllSlotsInfo = std::vector<TSlotInfo>;
// Type to address TAllSlotsInfo
using TIndex = ui16;
// Size in AppendBlockSize -> index in TAllSlotsInfo
- using TSearchTable = TVector<TIndex>;
- // Idx that indicates there is no record for it in TAllSlotsInfo
- static constexpr TIndex NoOpIdx = Max<TIndex>();
+ using TSearchTable = std::vector<TIndex>;
-
- THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
+ THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
const TSlotInfo *GetSlotInfo(ui32 size) const;
ui32 AlignByBlockSize(ui32 size) const;
void Output(IOutputStream &str) const;
@@ -55,6 +52,7 @@ namespace NKikimr {
private:
const ui32 AppendBlockSize;
+ const ui32 MinHugeBlobInBlocks;
TAllSlotsInfo AllSlotsInfo;
TSearchTable SearchTable;
};
@@ -68,7 +66,7 @@ namespace NKikimr {
const bool AddHeader;
// check whether this NEW blob is huge one; userPartSize doesn't include any metadata stored along with blob
- bool IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minREALHugeBlobInBytes) const;
+ bool IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minHugeBlobInBytes) const;
THugeBlobCtx(const std::shared_ptr<const THugeSlotsMap> &hugeSlotsMap, bool addHeader)
: HugeSlotsMap(hugeSlotsMap)
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
index 2aa98ea92c..699f5ca6ef 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
@@ -17,7 +17,6 @@ namespace NKikimr {
Y_UNIT_TEST(SerializeParse) {
ui32 chunkSize = 134274560u;
ui32 appendBlockSize = 56896u;
- ui32 minHugeBlobInBytes = 512u << 10u;
ui32 milestoneHugeBlobInBytes = 512u << 10u;
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
@@ -29,8 +28,8 @@ namespace NKikimr {
auto vctx = MakeIntrusive<TVDiskContext>(TActorId(), info->PickTopology(), counters, TVDiskID(0, 1, 0, 0, 0),
nullptr, NPDisk::DEVICE_TYPE_UNKNOWN);
std::unique_ptr<THullHugeKeeperPersState> state(
- new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize, appendBlockSize,
- minHugeBlobInBytes, milestoneHugeBlobInBytes, maxBlobInBytes,
+ new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize,
+ appendBlockSize, milestoneHugeBlobInBytes, maxBlobInBytes,
overhead, freeChunksReservation, logf));
state->LogPos = THullHugeRecoveryLogPos(0, 0, 100500, 50000, 70000, 56789, 39482);
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp
index be27874e3f..a6ff46e107 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp
@@ -2,6 +2,8 @@
#include <library/cpp/monlib/service/pages/templates.h>
+#include <ranges>
+
namespace NKikimr {
namespace NHuge {
@@ -24,7 +26,7 @@ namespace NKikimr {
}
void TChainLayoutBuilder::Output(IOutputStream &str, ui32 appendBlockSize) const {
- str << "CHAIN TABLE (MilesoneId=" << MilesoneId << " rows=" << Layout.size() << "):\n";
+ str << "CHAIN TABLE (MilestoneId=" << MilestoneId << " rows=" << Layout.size() << "):\n";
for (const auto &x : Layout) {
str << "Blocks# (" << x.Left << ", " << x.Right << "]";
if (appendBlockSize) {
@@ -64,7 +66,7 @@ namespace NKikimr {
}
void TChainLayoutBuilder::BuildUpward(ui32 left, ui32 right, ui32 overhead) {
- MilesoneId = Layout.size();
+ MilestoneId = Layout.size();
ui32 valBlocks = left;
ui32 shiftBlocks = 0;
@@ -83,6 +85,21 @@ namespace NKikimr {
////////////////////////////////////////////////////////////////////////////
// TChain
////////////////////////////////////////////////////////////////////////////
+ THugeSlot TChain::Convert(const NPrivate::TChunkSlot& id) const {
+ return THugeSlot(id.GetChunkId(), id.GetSlotId() * SlotSize, SlotSize);
+ }
+
+ NPrivate::TChunkSlot TChain::Convert(const TDiskPart& addr) const {
+ ui32 slotId = addr.Offset / SlotSize;
+ Y_VERIFY_S(slotId * SlotSize == addr.Offset, VDiskLogPrefix << "slotId# " << slotId
+ << " addr# " << addr.ToString() << " State# " << ToString());
+ return NPrivate::TChunkSlot(addr.ChunkIdx, slotId);
+ }
+
+ NPrivate::TChunkSlot TChain::Convert(const THugeSlot& slot) const {
+ return Convert(slot.GetDiskPart());
+ }
+
TMask TChain::BuildConstMask(const TString &prefix, ui32 slotsInChunk) {
Y_VERIFY_S(1 < slotsInChunk && slotsInChunk <= MaxNumberOfSlots,
prefix << "It's not a good idea to have so many slots in chunk;"
@@ -250,73 +267,114 @@ namespace NKikimr {
void TChain::Save(IOutputStream *s) const {
::Save(s, SlotsInChunk);
::Save(s, AllocatedSlots);
- if (LockedChunks) {
- TFreeSpace temp(FreeSpace);
- temp.insert(LockedChunks.begin(), LockedChunks.end());
- ::Save(s, temp);
- } else {
- ::Save(s, FreeSpace);
- }
+ ::SaveSize(s, FreeSpace.size() + LockedChunks.size());
+ ForEachFreeSpaceChunk(std::bind(&::Save<TFreeSpace::value_type>, s, std::placeholders::_1));
}
- void TChain::Load(IInputStream *s) {
- FreeSpace.clear();
- ui32 slotsInChunk = 0;
+ TChain TChain::Load(IInputStream *s, TString vdiskLogPrefix, ui32 appendBlockSize, ui32 blocksInChunk,
+ std::span<TChain> chains, bool *compatible) {
+ ui32 slotsInChunk;
::Load(s, slotsInChunk);
- Y_VERIFY_S(slotsInChunk == SlotsInChunk, VDiskLogPrefix
- << "slotsInChunk# " << slotsInChunk << " SlotsInChunk# " << SlotsInChunk);
- ::Load(s, AllocatedSlots);
- ::Load(s, FreeSpace);
- FreeSlotsInFreeSpace = 0;
- for (const auto &[chunkId, mask] : FreeSpace) {
- // all 1 in mask -- free slots
- // 0 - slot is in use
- FreeSlotsInFreeSpace += mask.Count();
+
+ const ui32 slotSizeInBlocks = blocksInChunk / slotsInChunk;
+ Y_ABORT_UNLESS(slotSizeInBlocks);
+ ui32 slotSize = slotSizeInBlocks * appendBlockSize; // assume optimal slot size for specific slots per chunk
+
+ // check if this goes with compatible chain
+ for (const TChain& chainFromBuilder : chains) {
+ if (chainFromBuilder.SlotsInChunk < slotsInChunk) { // no such chain from builder
+ *compatible = false;
+ } else if (chainFromBuilder.SlotsInChunk == slotsInChunk) {
+ slotSize = chainFromBuilder.SlotSize;
+ } else {
+ continue;
+ }
+ break;
+ }
+
+ TChain res{
+ std::move(vdiskLogPrefix),
+ slotsInChunk,
+ slotSize, // in bytes
+ };
+
+ ::Load(s, res.AllocatedSlots);
+ ::Load(s, res.FreeSpace);
+ for (const auto& [chunkId, mask] : res.FreeSpace) {
+ res.FreeSlotsInFreeSpace += mask.Count();
}
+
+ return res;
}
bool TChain::HaveBeenUsed() const {
- return AllocatedSlots != 0 || !FreeSpace.empty();
+ return AllocatedSlots; // chain is considered to be used if it contains any allocated slots
}
TString TChain::ToString() const {
TStringStream str;
- auto output = [&str] (const TFreeSpace &c) {
- for (const auto &x : c) {
- for (size_t i = 0; i < x.second.Size(); i++) {
- if (x.second.Test(i))
- str << " [" << x.first << " " << i << "]";
+
+ str << "{" << SlotSize << '/' << SlotsInChunk << " AllocatedSlots# " << AllocatedSlots << " Free#";
+
+ bool any = false;
+ ForEachFreeSpaceChunk([&](const auto& value) {
+ const auto& [chunk, bitmap] = value;
+ str << " {" << chunk;
+ ui32 begin;
+ ui32 prev = Max<ui32>();
+ Y_FOR_EACH_BIT(i, bitmap) {
+ if (prev == Max<ui32>()) {
+ begin = i;
+ } else if (i != prev + 1) {
+ str << ' ' << begin;
+ if (begin != prev) {
+ str << '-' << prev;
+ }
+ begin = prev;
}
+ prev = i;
}
- };
+ str << ' ' << begin;
+ if (begin != prev) {
+ str << '-' << prev;
+ }
+ str << '}';
+
+ any = true;
+ });
+
+ if (!any) {
+ str << " none";
+ }
- str << "{AllocatedSlots# " << AllocatedSlots << " [ChunkId FreeSlot]:";
- output(FreeSpace);
- output(LockedChunks);
str << "}";
return str.Str();
}
void TChain::RenderHtml(IOutputStream &str) const {
- auto output = [&str] (const TFreeSpace &c) {
- for (const auto &x : c) {
- size_t freeSlots = 0;
- for (size_t i = 0; i < x.second.Size(); i++) {
- if (x.second.Test(i))
- ++freeSlots;
+ HTML(str) {
+ TABLER() {
+ TABLED() {
+ str << SlotSize << "/" << SlotsInChunk;
}
- if (freeSlots) {
- str << " [" << x.first << " " << freeSlots << "]";
+ TABLED() {
+ ForEachFreeSpaceChunk([&](const auto& value) {
+ const auto& [chunk, bitmap] = value;
+ str << " [" << chunk << " " << bitmap.Count() << "]";
+ });
}
}
- };
-
- output(FreeSpace);
- output(LockedChunks);
+ }
}
- ui32 TChain::GetAllocatedSlots() const {
- return AllocatedSlots;
+ void TChain::RenderHtmlForUsage(IOutputStream &str) const {
+ HTML(str) {
+ TABLER() {
+ TABLED() { str << SlotSize; }
+ TABLED() { str << SlotsInChunk; }
+ TABLED() { str << AllocatedSlots; }
+ }
+ }
}
void TChain::GetOwnedChunks(TSet<TChunkIdx>& chunks) const {
@@ -330,87 +388,6 @@ namespace NKikimr {
}
////////////////////////////////////////////////////////////////////////////
- // TChainDelegator
- ////////////////////////////////////////////////////////////////////////////
- TChainDelegator::TChainDelegator(const TString &vdiskLogPrefix, ui32 valBlocks, ui32 shiftBlocks,
- ui32 chunkSize, ui32 appendBlockSize)
- : VDiskLogPrefix(vdiskLogPrefix)
- , Blocks(valBlocks)
- , ShiftInBlocks(shiftBlocks)
- , SlotsInChunk(0)
- , SlotSize(0)
- {
- ui32 slotSizeInBlocks = Blocks + ShiftInBlocks;
- ui32 blocksInChunk = chunkSize / appendBlockSize;
- Y_VERIFY_S(appendBlockSize * blocksInChunk == chunkSize, VDiskLogPrefix
- << "Blocks# " << Blocks << " ShiftInBlocks# " << ShiftInBlocks
- << " chunkSize# " << chunkSize << " appendBlockSize# " << appendBlockSize);
-
- SlotsInChunk = blocksInChunk / slotSizeInBlocks;
- SlotSize = slotSizeInBlocks * appendBlockSize;
-
- ChainPtr = MakeIntrusive<TChain>(vdiskLogPrefix, SlotsInChunk);
- }
-
- THugeSlot TChainDelegator::Convert(const NPrivate::TChunkSlot &id) const {
- return THugeSlot(id.GetChunkId(), id.GetSlotId() * SlotSize, SlotSize);
- }
-
- NPrivate::TChunkSlot TChainDelegator::Convert(const TDiskPart &addr) const {
- ui32 slotId = addr.Offset / SlotSize;
- Y_VERIFY_S(slotId * SlotSize == addr.Offset, VDiskLogPrefix
- << "slotId# " << slotId << " addr# " << addr.ToString() << " State# " << ToString());
- return NPrivate::TChunkSlot(addr.ChunkIdx, slotId);
- }
-
- NPrivate::TChunkSlot TChainDelegator::Convert(const THugeSlot &slot) const {
- return Convert(slot.GetDiskPart());
- }
-
- void TChainDelegator::Save(IOutputStream *s) const {
- ::Save(s, *ChainPtr);
- }
-
- void TChainDelegator::Load(IInputStream *s) {
- ::Load(s, *ChainPtr);
- }
-
- bool TChainDelegator::HaveBeenUsed() const {
- return ChainPtr->HaveBeenUsed();
- }
-
- TString TChainDelegator::ToString() const {
- TStringStream str;
- str << "{[SlotSize, SlotsInChunk]: [" << SlotSize << ", " << SlotsInChunk << "] "
- << ChainPtr->ToString() << "}";
- return str.Str();
- }
-
- void TChainDelegator::GetOwnedChunks(TSet<TChunkIdx>& chunks) const {
- ChainPtr->GetOwnedChunks(chunks);
- }
-
- void TChainDelegator::RenderHtml(IOutputStream &str) const {
- HTML(str) {
- TABLER() {
- TABLED() {str << SlotSize << " / " << SlotsInChunk;}
- TABLED() {ChainPtr->RenderHtml(str);}
- }
- }
- }
-
- void TChainDelegator::RenderHtmlForUsage(IOutputStream &str) const {
- HTML(str) {
- TABLER() {
- TABLED() {str << SlotSize;}
- TABLED() {str << SlotsInChunk;}
- TABLED() {str << ChainPtr->GetAllocatedSlots();}
- }
- }
- }
-
-
- ////////////////////////////////////////////////////////////////////////////
// TAllChains
////////////////////////////////////////////////////////////////////////////
TAllChains::TAllChains(
@@ -418,7 +395,6 @@ namespace NKikimr {
ui32 chunkSize,
ui32 appendBlockSize,
ui32 minHugeBlobInBytes,
- ui32 oldMinHugeBlobSizeInBytes,
ui32 milestoneBlobInBytes,
ui32 maxBlobInBytes,
ui32 overhead)
@@ -426,143 +402,137 @@ namespace NKikimr {
, ChunkSize(chunkSize)
, AppendBlockSize(appendBlockSize)
, MinHugeBlobInBytes(minHugeBlobInBytes)
- , OldMinHugeBlobSizeInBytes(oldMinHugeBlobSizeInBytes)
, MilestoneBlobInBytes(milestoneBlobInBytes)
, MaxBlobInBytes(maxBlobInBytes)
, Overhead(overhead)
+ , MinHugeBlobInBlocks(MinHugeBlobInBytes / AppendBlockSize)
{
- Y_VERIFY_S(MinHugeBlobInBytes != 0 &&
- MinHugeBlobInBytes >= AppendBlockSize &&
+ Y_VERIFY_S(MinHugeBlobInBytes &&
MinHugeBlobInBytes <= MilestoneBlobInBytes &&
- MinHugeBlobInBytes <= OldMinHugeBlobSizeInBytes &&
MilestoneBlobInBytes < MaxBlobInBytes, "INVALID CONFIGURATION! (SETTINGS ARE:"
- << " MaxBlobInBytes# " << MaxBlobInBytes << " MinHugeBlobInBytes# " << MinHugeBlobInBytes << " OldMinHugeBlobSizeInBytes# " << OldMinHugeBlobSizeInBytes
+ << " MaxBlobInBytes# " << MaxBlobInBytes << " MinHugeBlobInBytes# " << MinHugeBlobInBytes
<< " MilestoneBlobInBytes# " << MilestoneBlobInBytes << " ChunkSize# " << ChunkSize
<< " AppendBlockSize# " << AppendBlockSize << ")");
- BuildLayout();
- }
-
- TChainDelegator *TAllChains::GetChain(ui32 size) {
- return SearchTable.at(SizeToBlocks(size));
- }
- const TChainDelegator *TAllChains::GetChain(ui32 size) const {
- return SearchTable.at(SizeToBlocks(size));
+ BuildChains();
}
- THeapStat TAllChains::GetStat() const {
- THeapStat stat;
- for (const auto &x : ChainDelegators) {
- stat += x.ChainPtr->GetStat();
+ TChain *TAllChains::GetChain(ui32 size) {
+ if (size < MinHugeBlobInBytes || GetEndBlocks() * AppendBlockSize < size) {
+ return nullptr;
}
- return stat;
+ const size_t index = SizeToBlocks(size) - MinHugeBlobInBlocks;
+ Y_DEBUG_ABORT_UNLESS(index < SearchTable.size());
+ const size_t chainIndex = SearchTable[index];
+ Y_DEBUG_ABORT_UNLESS(chainIndex < Chains.size());
+ return &Chains[chainIndex];
}
- void TAllChains::PrintOutChains(IOutputStream &str) const {
- str << "CHAIN TABLE (rows=" << ChainDelegators.size() << "):\n";
- for (const auto &x : ChainDelegators) {
- str << "Blocks# (" << x.Blocks << ", " << (x.Blocks + x.ShiftInBlocks) << "] "
- << "Bytes# (" << (x.Blocks * AppendBlockSize) << ", "
- << ((x.Blocks + x.ShiftInBlocks) * AppendBlockSize) << "] "
- << " SlotSize# " << x.SlotSize << " SlotsInChunk# " << x.SlotsInChunk << "\n";
+ const TChain *TAllChains::GetChain(ui32 size) const {
+ if (size < MinHugeBlobInBytes || GetEndBlocks() * AppendBlockSize < size) {
+ return nullptr;
}
+ Y_ABORT_UNLESS(MinHugeBlobInBytes <= size);
+ const size_t index = SizeToBlocks(size) - MinHugeBlobInBlocks;
+ Y_DEBUG_ABORT_UNLESS(index < SearchTable.size());
+ const size_t chainIndex = SearchTable[index];
+ Y_DEBUG_ABORT_UNLESS(chainIndex < Chains.size());
+ return &Chains[chainIndex];
}
- void TAllChains::PrintOutSearchTable(IOutputStream &str) const {
- str << "SEARCH TABLE:\n";
- for (const auto &x : SearchTable) {
- str << (&x - &SearchTable[0]) << " idx: ";
- if (x) {
- str << x - &ChainDelegators[0];
- } else {
- str << "null";
- }
- str << "\n";
+ THeapStat TAllChains::GetStat() const {
+ THeapStat stat;
+ for (const auto& chain : Chains) {
+ stat += chain.GetStat();
}
+ return stat;
}
void TAllChains::Save(IOutputStream *s) const {
- if (StartMode == EStartMode::Loaded) {
- ui32 size = ChainDelegators.size();
- ::Save(s, size);
- for (auto& d : ChainDelegators) {
- ::Save(s, d);
- }
- } else {
- std::vector<const TChainDelegator*> delegators;
- for (auto &x : ChainDelegators) {
- if (!x.HaveBeenUsed() && x.SlotSize < FirstLoadedSlotSize) {
- continue; // preserving backward compatibility until no allocations
+ // check if we can write compatible entrypoint (with exactly the same set of chains)
+ bool writeCompatible = true;
+ ui32 numChains = 0;
+ if (DeserializedSlotSizes.empty()) { // if this was initially empty heap, write it fully
+ writeCompatible = false;
+ } else {
+ for (auto& chain : Chains) {
+ if (DeserializedSlotSizes.contains(chain.SlotSize)) {
+ ++numChains;
+ } else if (chain.HaveBeenUsed()) {
+ writeCompatible = false;
+ break;
}
- delegators.emplace_back(&x);
}
+ }
- ui32 size = delegators.size();
- ::Save(s, size);
- for (auto x : delegators) {
- ::Save(s, *x);
- }
+ if (!writeCompatible) { // we can't, so we serialize all our chains anyway
+ numChains = Chains.size();
}
+ ::Save(s, numChains);
+
+ // serialize selected chains
+ auto chains = Chains | std::views::filter([&](const auto& chain) {
+ return !writeCompatible || DeserializedSlotSizes.contains(chain.SlotSize);
+ });
+ ::SaveRange(s, chains.begin(), chains.end());
}
void TAllChains::Load(IInputStream *s) {
- ui32 size = 0;
- // load array size
- ::Load(s, size);
- if (size == ChainDelegators.size()) {
- StartMode = EStartMode::Loaded;
- // load map and current map are of the same size, just load it
- for (auto &x : ChainDelegators) {
- ::Load(s, x);
- }
- } else if (size < ChainDelegators.size()) {
- // map size has been changed, run migration
- StartMode = EStartMode::Migrated;
- TAllChainDelegators chainDelegators = BuildChains(OldMinHugeBlobSizeInBytes);
- Y_VERIFY_S(size > 0 && size == chainDelegators.size(), "size# " << size
- << " chainDelegators.size()# " << chainDelegators.size());
-
- // load into temporary delegators
- for (auto &x : chainDelegators) {
- ::Load(s, x);
- }
+ std::vector<TChain> newChains;
+ newChains.reserve(Chains.size());
+
+ auto chainsIt = Chains.begin();
+ const auto chainsEnd = Chains.end();
+
+ Y_DEBUG_ABORT_UNLESS(ChunkSize % AppendBlockSize == 0);
+ Y_DEBUG_ABORT_UNLESS(AppendBlockSize <= ChunkSize);
+ const ui32 blocksInChunk = ChunkSize / AppendBlockSize;
+
+ bool compatible = true; // if the entrypoint is compatible with chain layout builder
+ ui32 prevSlotSize = 0;
+
+ ui32 size;
+ for (::Load(s, size); size; --size) {
+ auto chain = TChain::Load(s, VDiskLogPrefix, AppendBlockSize, blocksInChunk, {chainsIt, chainsEnd},
+ &compatible);
+ DeserializedSlotSizes.insert(chain.SlotSize);
- // migrate
- using TIt = TAllChainDelegators::iterator;
- TIt loadedIt = chainDelegators.begin();
- TIt loadedEnd = chainDelegators.end();
- FirstLoadedSlotSize = loadedIt->SlotSize;
- for (TIt it = ChainDelegators.begin(); it != ChainDelegators.end(); ++it) {
- Y_ABORT_UNLESS(loadedIt != loadedEnd);
- if (loadedIt->SlotSize == it->SlotSize) {
- *it = std::move(*loadedIt);
- ++loadedIt;
+ Y_ABORT_UNLESS(chain.SlotSize > std::exchange(prevSlotSize, chain.SlotSize));
+
+ for (; chainsIt != chainsEnd && chainsIt->SlotSize <= chain.SlotSize; ++chainsIt) {
+ if (chainsIt->SlotSize == chain.SlotSize) {
+ Y_ABORT_UNLESS(chain.SlotsInChunk == chainsIt->SlotsInChunk);
+ Y_ABORT_UNLESS(!chainsIt->HaveBeenUsed());
+ } else {
+ newChains.push_back(std::move(*chainsIt));
}
}
- Y_ABORT_UNLESS(loadedIt == loadedEnd);
- } else {
- // entry point size rollback case
- Y_ABORT_UNLESS(size > ChainDelegators.size());
- ui32 curChainDelegatorsSize = ChainDelegators.size();
- Y_FAIL_S("Impossible case; MinHugeBlobInBytes# " << MinHugeBlobInBytes
- << " MilestoneBlobInBytes# " << MilestoneBlobInBytes
- << " loadedSize# " << size
- << " curChainDelegatorsSize# " << curChainDelegatorsSize);
+
+ newChains.push_back(std::move(chain));
+ }
+
+ std::ranges::move(chainsIt, chainsEnd, std::back_inserter(newChains));
+
+ Chains = std::move(newChains);
+ if (!compatible) { // deserialized slot sizes can't be stored in compatible way
+ DeserializedSlotSizes.clear();
}
}
void TAllChains::GetOwnedChunks(TSet<TChunkIdx>& chunks) const {
- for (const TChainDelegator& delegator : ChainDelegators) {
- delegator.GetOwnedChunks(chunks);
+ for (const TChain& chain : Chains) {
+ chain.GetOwnedChunks(chunks);
}
}
TString TAllChains::ToString() const {
TStringStream str;
- str << "{ChunkSize# " << ChunkSize << " AppendBlockSize# " << AppendBlockSize
- << " MinHugeBlobInBytes# " << MinHugeBlobInBytes << " MaxBlobInBytes# " << MaxBlobInBytes;
- for (const auto & x : ChainDelegators) {
- str << " {CHAIN " << x.ToString() << "}";
+ str << "{ChunkSize# " << ChunkSize
+ << " AppendBlockSize# " << AppendBlockSize
+ << " MinHugeBlobInBytes# " << MinHugeBlobInBytes
+ << " MaxBlobInBytes# " << MaxBlobInBytes;
+ for (const auto& chain : Chains) {
+ str << " {CHAIN " << chain.ToString() << "}";
}
str << "}";
return str.Str();
@@ -578,8 +548,9 @@ namespace NKikimr {
}
}
TABLEBODY() {
- for (const auto & x : ChainDelegators)
- x.RenderHtml(str);
+ for (const auto& chain : Chains) {
+ chain.RenderHtml(str);
+ }
}
}
}
@@ -596,8 +567,9 @@ namespace NKikimr {
}
}
TABLEBODY() {
- for (const auto & x : ChainDelegators)
- x.RenderHtmlForUsage(str);
+ for (const auto& chain : Chains) {
+ chain.RenderHtmlForUsage(str);
+ }
}
}
}
@@ -605,91 +577,87 @@ namespace NKikimr {
TVector<NPrivate::TChainLayoutBuilder::TSeg> TAllChains::GetLayout() const {
TVector<NPrivate::TChainLayoutBuilder::TSeg> res;
- res.reserve(ChainDelegators.size());
- for (const auto &x : ChainDelegators) {
- res.push_back(NPrivate::TChainLayoutBuilder::TSeg {x.Blocks, x.Blocks + x.ShiftInBlocks} );
+ res.reserve(Chains.size());
+ ui32 prevSlotSizeInBlocks = MinHugeBlobInBlocks;
+ for (const auto& chain : Chains) {
+ const ui32 slotSizeInBlocks = chain.SlotSize / AppendBlockSize;
+ res.push_back({
+ prevSlotSizeInBlocks,
+ slotSizeInBlocks - prevSlotSizeInBlocks,
+ });
+ prevSlotSizeInBlocks = slotSizeInBlocks;
}
return res;
}
std::shared_ptr<THugeSlotsMap> TAllChains::BuildHugeSlotsMap() const {
- THugeSlotsMap::TAllSlotsInfo targetAllSlotsInfo;
- THugeSlotsMap::TSearchTable targetSearchTable;
-
- for (const auto &x : SearchTable) {
- if (!x) {
- // first records in SearchTable are equal to nullptr
- targetSearchTable.push_back(THugeSlotsMap::NoOpIdx);
- continue;
- }
-
- if (targetAllSlotsInfo.empty() || targetAllSlotsInfo.back().SlotSize != x->SlotSize) {
- targetAllSlotsInfo.emplace_back(x->SlotSize, x->SlotsInChunk);
- }
- targetSearchTable.push_back(THugeSlotsMap::TIndex(targetAllSlotsInfo.size() - 1));
+ THugeSlotsMap::TAllSlotsInfo allSlotsInfo;
+ for (const auto& chain : Chains) {
+ allSlotsInfo.emplace_back(chain.SlotSize, chain.SlotsInChunk);
}
-
- return std::make_shared<THugeSlotsMap>(AppendBlockSize, std::move(targetAllSlotsInfo),
- std::move(targetSearchTable));
+ return std::make_shared<THugeSlotsMap>(AppendBlockSize, MinHugeBlobInBlocks, std::move(allSlotsInfo),
+ THugeSlotsMap::TSearchTable(SearchTable));
}
////////////////////////////////////////////////////////////////////////////
// TAllChains: Private
////////////////////////////////////////////////////////////////////////////
- TAllChains::TAllChainDelegators TAllChains::BuildChains(ui32 minHugeBlobInBytes) const {
- // minHugeBlobInBytes -- is the only variable parameter, used for migration
- const ui32 startBlocks = minHugeBlobInBytes / AppendBlockSize;
- const ui32 mileStoneBlocks = MilestoneBlobInBytes / AppendBlockSize;
+ void TAllChains::BuildChains() {
+ const ui32 startBlocks = MinHugeBlobInBlocks;
+ const ui32 milestoneBlocks = MilestoneBlobInBytes / AppendBlockSize;
const ui32 endBlocks = GetEndBlocks();
- NPrivate::TChainLayoutBuilder builder(startBlocks, mileStoneBlocks, endBlocks, Overhead);
- Y_ABORT_UNLESS(!builder.GetLayout().empty());
+ NPrivate::TChainLayoutBuilder builder(startBlocks, milestoneBlocks, endBlocks, Overhead);
+ const ui32 blocksInChunk = ChunkSize / AppendBlockSize;
- TAllChainDelegators result;
for (auto x : builder.GetLayout()) {
- result.emplace_back(VDiskLogPrefix, x.Left, x.Right - x.Left,
- ChunkSize, AppendBlockSize);
+ const ui32 slotSizeInBlocks = x.Right;
+ const ui32 slotSize = slotSizeInBlocks * AppendBlockSize;
+ const ui32 slotsInChunk = blocksInChunk / slotSizeInBlocks;
+ Chains.emplace_back(VDiskLogPrefix, slotsInChunk, slotSize);
}
- return result;
+
+ Y_ABORT_UNLESS(!Chains.empty());
}
void TAllChains::BuildSearchTable() {
- const ui32 endBlocks = GetEndBlocks();
- Y_DEBUG_ABORT_UNLESS(!ChainDelegators.empty());
- TAllChainDelegators::iterator it = ChainDelegators.begin();
- TChainDelegator *ptr = nullptr;
- ui32 blocks = it->Blocks;
- for (ui32 i = 0; i <= endBlocks; i++) {
- if (i <= blocks) {
- } else {
- ptr = &(*it);
+ Y_ABORT_UNLESS(SearchTable.empty());
+ Y_ABORT_UNLESS(!Chains.empty());
+
+ const ui32 startBlocks = MinHugeBlobInBlocks;
+ const ui32 minSize = startBlocks * AppendBlockSize;
+ const ui32 endBlocks = GetEndBlocks(); // maximum possible number of blocks per huge blob
+ auto it = Chains.begin();
+ ui16 index = 0;
+
+ SearchTable.reserve(endBlocks - startBlocks + 1);
+ for (ui32 i = startBlocks, size = minSize; i <= endBlocks; ++i, size += AppendBlockSize) {
+ if (it->SlotSize < size) { // size doesn't fit in current chain, but it must fit into next one
++it;
- if (it == ChainDelegators.end())
- blocks = ui32(-1);
- else
- blocks = it->Blocks;
+ Y_ABORT_UNLESS(it != Chains.end());
+ Y_ABORT_UNLESS(size <= it->SlotSize);
+ Y_ABORT_UNLESS(index != Max<ui16>());
+ ++index;
}
- SearchTable.push_back(ptr);
+ SearchTable.push_back(index);
}
}
- void TAllChains::BuildLayout()
- {
- ChainDelegators = BuildChains(MinHugeBlobInBytes);
- Y_ABORT_UNLESS(!ChainDelegators.empty());
- BuildSearchTable();
+ ui32 TAllChains::SizeToBlocks(ui32 size) const {
+ return (size + AppendBlockSize - 1) / AppendBlockSize;
}
- inline ui32 TAllChains::SizeToBlocks(ui32 size) const {
- ui32 sizeInBlocks = size / AppendBlockSize;
- sizeInBlocks += !(sizeInBlocks * AppendBlockSize == size);
- return sizeInBlocks;
+ ui32 TAllChains::GetEndBlocks() const {
+ return SizeToBlocks(MaxBlobInBytes);
}
- inline ui32 TAllChains::GetEndBlocks() const {
- ui32 endBlocks = MaxBlobInBytes / AppendBlockSize;
- endBlocks += !(endBlocks * AppendBlockSize == MaxBlobInBytes);
- return endBlocks;
+ void TAllChains::FinishRecovery() {
+ ui32 prevSlotSize = 0;
+ for (const TChain& chain : Chains) {
+ Y_ABORT_UNLESS(prevSlotSize < chain.SlotSize);
+ prevSlotSize = chain.SlotSize;
+ }
+ BuildSearchTable();
}
////////////////////////////////////////////////////////////////////////////
@@ -701,7 +669,6 @@ namespace NKikimr {
ui32 chunkSize,
ui32 appendBlockSize,
ui32 minHugeBlobInBytes,
- ui32 oldMinHugeBlobSizeInBytes,
ui32 mileStoneBlobInBytes,
ui32 maxBlobInBytes,
ui32 overhead,
@@ -709,42 +676,41 @@ namespace NKikimr {
: VDiskLogPrefix(vdiskLogPrefix)
, FreeChunksReservation(freeChunksReservation)
, FreeChunks()
- , Chains(vdiskLogPrefix, chunkSize, appendBlockSize, minHugeBlobInBytes, oldMinHugeBlobSizeInBytes,
- mileStoneBlobInBytes, maxBlobInBytes, overhead)
+ , Chains(vdiskLogPrefix, chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ maxBlobInBytes, overhead)
{}
//////////////////////////////////////////////////////////////////////////////////////////
// THeap: main functions
//////////////////////////////////////////////////////////////////////////////////////////
- THugeSlot THeap::ConvertDiskPartToHugeSlot(const TDiskPart &addr) const {
- const TChainDelegator *chainD = Chains.GetChain(addr.Size);
- Y_VERIFY_S(chainD && (addr.Offset / chainD->SlotSize * chainD->SlotSize == addr.Offset), VDiskLogPrefix
- << "chainD# " << (chainD ? chainD->ToString() : "nullptr") << " addr# " << addr.ToString());
- return THugeSlot(addr.ChunkIdx, addr.Offset, chainD->SlotSize);
+ THugeSlot THeap::ConvertDiskPartToHugeSlot(const TDiskPart& addr) const {
+ const TChain *chain = Chains.GetChain(addr.Size);
+ Y_ABORT_UNLESS(chain);
+ return chain->Convert(chain->Convert(addr));
}
bool THeap::Allocate(ui32 size, THugeSlot *hugeSlot, ui32 *slotSize) {
- TChainDelegator *chainD = Chains.GetChain(size);
- Y_VERIFY_S(chainD, VDiskLogPrefix << "size# " << size << " Heap# " << ToString());
- *slotSize = chainD->SlotSize;
+ TChain *chain = Chains.GetChain(size);
+ Y_VERIFY_S(chain, VDiskLogPrefix << "size# " << size << " Heap# " << ToString());
+ *slotSize = chain->SlotSize;
NPrivate::TChunkSlot id;
- if (!chainD->ChainPtr->Allocate(&id)) { // no available slot in free space of the chain
+ if (!chain->Allocate(&id)) { // no available slot in free space of the chain
if (FreeChunks.empty()) { // no free chunks left for reuse -- request a new chunk
return false;
}
- chainD->ChainPtr->Allocate(&id, GetChunkIdFromFreeChunks()); // reuse free chunk for this chain
+ chain->Allocate(&id, GetChunkIdFromFreeChunks()); // reuse free chunk for this chain
}
- *hugeSlot = chainD->Convert(id);
+ *hugeSlot = chain->Convert(id);
return true;
}
TFreeRes THeap::Free(const TDiskPart &addr) {
ui32 size = addr.Size;
- TChainDelegator *chainD = Chains.GetChain(size);
- Y_ABORT_UNLESS(chainD);
+ TChain *chain = Chains.GetChain(size);
+ Y_ABORT_UNLESS(chain);
- TFreeRes res = chainD->ChainPtr->Free(chainD->Convert(addr));
+ TFreeRes res = chain->Free(chain->Convert(addr));
if (res.ChunkId) {
PutChunkIdToFreeChunks(res.ChunkId);
}
@@ -766,13 +732,19 @@ namespace NKikimr {
}
bool THeap::LockChunkForAllocation(ui32 chunkId, ui32 slotSize) {
- TChainDelegator *cd = Chains.GetChain(slotSize);
- return cd->ChainPtr->LockChunkForAllocation(chunkId);
+ TChain *chain = Chains.GetChain(slotSize);
+ Y_ABORT_UNLESS(chain);
+ return chain->LockChunkForAllocation(chunkId);
}
void THeap::UnlockChunk(ui32 chunkId, ui32 slotSize) {
- TChainDelegator *cd = Chains.GetChain(slotSize);
- cd->ChainPtr->UnlockChunk(chunkId);
+ TChain *chain = Chains.GetChain(slotSize);
+ Y_ABORT_UNLESS(chain);
+ chain->UnlockChunk(chunkId);
+ }
+
+ void THeap::FinishRecovery() {
+ Chains.FinishRecovery();
}
THeapStat THeap::GetStat() const {
@@ -788,11 +760,11 @@ namespace NKikimr {
void THeap::RecoveryModeAllocate(const TDiskPart &addr) {
ui32 size = addr.Size;
- TChainDelegator *chainD = Chains.GetChain(size);
- Y_VERIFY_S(chainD, VDiskLogPrefix << "State# " << ToString());
+ TChain *chain = Chains.GetChain(size);
+ Y_VERIFY_S(chain, VDiskLogPrefix << "State# " << ToString());
- NPrivate::TChunkSlot id(chainD->Convert(addr));
- bool allocated = chainD->ChainPtr->RecoveryModeAllocate(id);
+ NPrivate::TChunkSlot id(chain->Convert(addr));
+ bool allocated = chain->RecoveryModeAllocate(id);
if (allocated) {
return;
} else {
@@ -800,7 +772,7 @@ namespace NKikimr {
TFreeChunks::iterator it = FreeChunks.find(chunkId);
Y_VERIFY_S(it != FreeChunks.end(), VDiskLogPrefix << "addr# " << addr.ToString() << " State# " << ToString());
FreeChunks.erase(it);
- chainD->ChainPtr->RecoveryModeAllocate(id, chunkId, false);
+ chain->RecoveryModeAllocate(id, chunkId, false);
}
}
@@ -817,9 +789,9 @@ namespace NKikimr {
}
bool THeap::ReleaseSlot(THugeSlot slot) {
- TChainDelegator* const chain = Chains.GetChain(slot.GetSize());
+ TChain* const chain = Chains.GetChain(slot.GetSize());
Y_VERIFY_S(chain, VDiskLogPrefix << "State# " << ToString() << " slot# " << slot.ToString());
- if (TFreeRes res = chain->ChainPtr->Free(chain->Convert(slot)); res.ChunkId) {
+ if (TFreeRes res = chain->Free(chain->Convert(slot)); res.ChunkId) {
PutChunkIdToFreeChunks(res.ChunkId);
return res.InLockedChunks;
}
@@ -827,12 +799,12 @@ namespace NKikimr {
}
void THeap::OccupySlot(THugeSlot slot, bool inLockedChunks) {
- TChainDelegator* const chain = Chains.GetChain(slot.GetSize());
+ TChain* const chain = Chains.GetChain(slot.GetSize());
Y_VERIFY_S(chain, VDiskLogPrefix << "State# " << ToString() << " slot# " << slot.ToString());
- if (!chain->ChainPtr->RecoveryModeAllocate(chain->Convert(slot))) {
+ if (!chain->RecoveryModeAllocate(chain->Convert(slot))) {
const size_t numErased = FreeChunks.erase(slot.GetChunkId());
Y_VERIFY_S(numErased, VDiskLogPrefix << "State# " << ToString() << " slot# " << slot.ToString());
- chain->ChainPtr->RecoveryModeAllocate(chain->Convert(slot), slot.GetChunkId(), inLockedChunks);
+ chain->RecoveryModeAllocate(chain->Convert(slot), slot.GetChunkId(), inLockedChunks);
}
}
@@ -899,9 +871,8 @@ namespace NKikimr {
TString THeap::ToString() const {
TStringStream str;
- str << "FreeChunks: ";
- str << FormatList(FreeChunks);
- str << " CHAINS: " << Chains.ToString();
+ str << "FreeChunks# " << FormatList(FreeChunks)
+ << " Chains# {" << Chains.ToString() << '}';
return str.Str();
}
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h
index 6fbae987e4..afdfe26959 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h
@@ -70,7 +70,7 @@ namespace NKikimr {
TChainLayoutBuilder(ui32 left, ui32 milestone, ui32 right, ui32 overhead);
const TVector<TSeg> &GetLayout() const { return Layout; }
- const TSeg &GetMilestoneSegment() const { return Layout.at(MilesoneId); }
+ const TSeg &GetMilestoneSegment() const { return Layout.at(MilestoneId); }
TString ToString(ui32 appendBlockSize = 0) const;
void Output(IOutputStream &str, ui32 appendBlockSize = 0) const;
@@ -81,7 +81,7 @@ namespace NKikimr {
TVector<TSeg> Layout;
// An index in Layout vector, where milestone segment starts
- size_t MilesoneId = Max<size_t>();
+ size_t MilestoneId = Max<size_t>();
};
} // NPrivate
@@ -90,29 +90,43 @@ namespace NKikimr {
// TChain
// It manages all slots of some fixed size.
////////////////////////////////////////////////////////////////////////////
- class TChain : public TThrRefBase {
+ class TChain {
using TChunkID = ui32;
using TFreeSpace = TMap<TChunkID, TMask>;
static constexpr ui32 MaxNumberOfSlots = 32768; // it's not a good idea to have more slots than this
- const TString VDiskLogPrefix;
- const ui32 SlotsInChunk;
- const TMask ConstMask; // mask of 'all slots are free'
+ /*const*/ TString VDiskLogPrefix;
+ /*const*/ TMask ConstMask; // mask of 'all slots are free'
TFreeSpace FreeSpace;
TFreeSpace LockedChunks;
ui32 AllocatedSlots = 0;
ui32 FreeSlotsInFreeSpace = 0;
public:
+ /*const*/ ui32 SlotsInChunk;
+ /*const*/ ui32 SlotSize;
+
+ public:
static TMask BuildConstMask(const TString &prefix, ui32 slotsInChunk);
public:
- TChain(const TString &vdiskLogPrefix, const ui32 slotsInChunk)
- : VDiskLogPrefix(vdiskLogPrefix)
- , SlotsInChunk(slotsInChunk)
+ TChain(TString vdiskLogPrefix, ui32 slotsInChunk, ui32 slotSize)
+ : VDiskLogPrefix(std::move(vdiskLogPrefix))
, ConstMask(BuildConstMask(vdiskLogPrefix, slotsInChunk))
+ , SlotsInChunk(slotsInChunk)
+ , SlotSize(slotSize)
{}
+ TChain(TChain&&) = default;
+ TChain(const TChain&) = delete;
+
+ TChain& operator=(TChain&&) = default;
+ TChain& operator=(const TChain&) = delete;
+
+ THugeSlot Convert(const NPrivate::TChunkSlot& id) const;
+ NPrivate::TChunkSlot Convert(const TDiskPart& addr) const;
+ NPrivate::TChunkSlot Convert(const THugeSlot& slot) const;
+
// returns true if allocated, false -- if no free slots
bool Allocate(NPrivate::TChunkSlot *id);
// allocate id, but we know that this chain doesn't have free slots, so add a chunk to it
@@ -126,71 +140,49 @@ namespace NKikimr {
bool RecoveryModeAllocate(const NPrivate::TChunkSlot &id);
void RecoveryModeAllocate(const NPrivate::TChunkSlot &id, TChunkID chunkId, bool inLockedChunks);
void Save(IOutputStream *s) const;
- void Load(IInputStream *s);
bool HaveBeenUsed() const;
TString ToString() const;
void RenderHtml(IOutputStream &str) const;
- ui32 GetAllocatedSlots() const;
+ void RenderHtmlForUsage(IOutputStream &str) const;
void GetOwnedChunks(TSet<TChunkIdx>& chunks) const;
- };
- using TChainPtr = TIntrusivePtr<TChain>;
-
- ////////////////////////////////////////////////////////////////////////////
- // TChainDelegator
- ////////////////////////////////////////////////////////////////////////////
- struct TChainDelegator {
- TString VDiskLogPrefix;
- ui32 Blocks;
- ui32 ShiftInBlocks;
- ui32 SlotsInChunk;
- ui32 SlotSize;
- TChainPtr ChainPtr;
-
- TChainDelegator(const TString &vdiskLogPrefix,
- ui32 valBlocks,
- ui32 shiftBlocks,
- ui32 chunkSize,
- ui32 appendBlockSize);
- TChainDelegator(TChainDelegator &&) = default;
- TChainDelegator &operator =(TChainDelegator &&) = default;
- TChainDelegator(const TChainDelegator &) = delete;
- TChainDelegator &operator =(const TChainDelegator &) = delete;
- THugeSlot Convert(const NPrivate::TChunkSlot &id) const;
- NPrivate::TChunkSlot Convert(const TDiskPart &addr) const;
- NPrivate::TChunkSlot Convert(const THugeSlot &slot) const;
- void Save(IOutputStream *s) const;
- void Load(IInputStream *s);
- bool HaveBeenUsed() const;
- TString ToString() const;
- void GetOwnedChunks(TSet<TChunkIdx>& chunks) const;
- void RenderHtml(IOutputStream &str) const;
- void RenderHtmlForUsage(IOutputStream &str) const;
+ static TChain Load(IInputStream *s, TString vdiskLogPrefix, ui32 appendBlockSize, ui32 blocksInChunk,
+ std::span<TChain> chains, bool *compatible);
+
+ template<typename T>
+ void ForEachFreeSpaceChunk(T&& callback) const {
+ auto freeIt = FreeSpace.begin();
+ const auto freeEnd = FreeSpace.end();
+ auto lockedIt = LockedChunks.begin();
+ const auto lockedEnd = LockedChunks.end();
+ while (freeIt != freeEnd || lockedIt != lockedEnd) {
+ if (lockedIt == lockedEnd || freeIt->first < lockedIt->first) {
+ std::invoke(callback, *freeIt++);
+ } else if (freeIt == freeEnd || lockedIt->first < freeIt->first) {
+ std::invoke(callback, *lockedIt++);
+ } else {
+ Y_ABORT("intersecting sets of keys for FreeSpace and LockedChunks");
+ }
+ }
+ }
};
-
////////////////////////////////////////////////////////////////////////////
// TAllChains
////////////////////////////////////////////////////////////////////////////
class TAllChains {
public:
- using TAllChainDelegators = TVector<TChainDelegator>;
- using TSearchTable = TVector<TChainDelegator*>;
-
TAllChains(const TString &vdiskLogPrefix,
ui32 chunkSize,
ui32 appendBlockSize,
ui32 minHugeBlobInBytes,
- ui32 oldMinHugeBlobSizeInBytes,
ui32 milestoneBlobInBytes,
ui32 maxBlobInBytes,
ui32 overhead);
// return a pointer to corresponding chain delegator by object byte size
- TChainDelegator *GetChain(ui32 size);
- const TChainDelegator *GetChain(ui32 size) const;
+ TChain *GetChain(ui32 size);
+ const TChain *GetChain(ui32 size) const;
THeapStat GetStat() const;
- void PrintOutChains(IOutputStream &str) const;
- void PrintOutSearchTable(IOutputStream &str) const;
void Save(IOutputStream *s) const;
void Load(IInputStream *s);
void GetOwnedChunks(TSet<TChunkIdx>& chunks) const;
@@ -199,39 +191,28 @@ namespace NKikimr {
void RenderHtmlForUsage(IOutputStream &str) const;
// for testing purposes
TVector<NPrivate::TChainLayoutBuilder::TSeg> GetLayout() const;
- // returns (ChainsSize, SearchTableSize)
- std::pair<ui32, ui32> GetTablesSize() const {
- return std::pair<ui32, ui32>(ChainDelegators.size(), SearchTable.size());
- }
// Builds a map of BlobSize -> THugeSlotsMap::TSlotInfo for THugeBlobCtx
std::shared_ptr<THugeSlotsMap> BuildHugeSlotsMap() const;
- private:
+ void FinishRecovery();
- TAllChainDelegators BuildChains(ui32 minHugeBlobInBytes) const;
+ private:
+ void BuildChains();
void BuildSearchTable();
- void BuildLayout();
inline ui32 SizeToBlocks(ui32 size) const;
inline ui32 GetEndBlocks() const;
- enum class EStartMode {
- Empty = 1,
- Loaded = 2,
- Migrated = 3,
- };
-
const TString VDiskLogPrefix;
const ui32 ChunkSize;
const ui32 AppendBlockSize;
const ui32 MinHugeBlobInBytes;
- const ui32 OldMinHugeBlobSizeInBytes;
const ui32 MilestoneBlobInBytes;
const ui32 MaxBlobInBytes;
const ui32 Overhead;
- EStartMode StartMode = EStartMode::Empty;
- ui32 FirstLoadedSlotSize = 0;
- TAllChainDelegators ChainDelegators;
- TSearchTable SearchTable;
+ const ui32 MinHugeBlobInBlocks;
+ THashSet<ui32> DeserializedSlotSizes; // a set of SlotSize values of recovered stream
+ std::vector<TChain> Chains;
+ std::vector<ui16> SearchTable; // (NumFullBlocks - 1) -> Chain index
};
@@ -255,7 +236,6 @@ namespace NKikimr {
ui32 appendBlockSize,
// min size of the huge blob
ui32 minHugeBlobInBytes,
- ui32 oldMinHugeBlobSizeInBytes,
// fixed point to calculate layout (for backward compatibility)
ui32 mileStoneBlobInBytes,
// max size of the blob
@@ -266,13 +246,13 @@ namespace NKikimr {
ui32 SlotNumberOfThisSize(ui32 size) const {
- const TChainDelegator *chainD = Chains.GetChain(size);
- return chainD ? chainD->SlotsInChunk : 0;
+ const TChain *chain = Chains.GetChain(size);
+ return chain ? chain->SlotsInChunk : 0;
}
ui32 SlotSizeOfThisSize(ui32 size) const {
- const TChainDelegator *chainD = Chains.GetChain(size);
- return chainD ? chainD->SlotSize : 0;
+ const TChain *chain = Chains.GetChain(size);
+ return chain ? chain->SlotSize : 0;
}
// Builds a map of BlobSize -> THugeSlotsMap::TSlotInfo for THugeBlobCtx
@@ -302,6 +282,7 @@ namespace NKikimr {
void RecoveryModeRemoveChunks(const TVector<ui32> &chunkIds);
bool ReleaseSlot(THugeSlot slot);
void OccupySlot(THugeSlot slot, bool inLockedChunks);
+ void FinishRecovery();
//////////////////////////////////////////////////////////////////////////////////////////
// Serialize/Parse/Check
@@ -317,10 +298,6 @@ namespace NKikimr {
void RenderHtml(IOutputStream &str) const;
TString ToString() const;
- void PrintOutSearchTable(IOutputStream &str) {
- Chains.PrintOutSearchTable(str);
- }
-
private:
inline ui32 GetChunkIdFromFreeChunks();
inline void PutChunkIdToFreeChunks(ui32 chunkId);
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp
index df27d118e9..865bf38513 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp
@@ -26,7 +26,6 @@ namespace NKikimr {
Contexts.GetVCtx(),
ChunkSize,
AppendBlockSize,
- AppendBlockSize,
cfg.MinHugeBlobInBytes,
cfg.MilestoneHugeBlobInBytes,
cfg.MaxLogoBlobDataSize,
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp
index 5de29e6f46..0edd937d24 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp
@@ -79,7 +79,7 @@ namespace NKikimr {
}
void AllocFreeOneChunk(ui32 slotsInChunk) {
- TChain chain("vdisk", slotsInChunk);
+ TChain chain("vdisk", slotsInChunk, 1);
TVector<NPrivate::TChunkSlot> arr;
AllocateScenaryOneChunk(chain, arr, slotsInChunk);
FreeScenaryOneChunk(chain, arr, slotsInChunk);
@@ -156,7 +156,7 @@ namespace NKikimr {
}
void AllocFreeAlloc(ui32 slotsInChunk) {
- TChain chain("vdisk", slotsInChunk);
+ TChain chain("vdisk", slotsInChunk, 1);
TVector<NPrivate::TChunkSlot> arr;
TVector<ui32> chunks;
@@ -171,17 +171,14 @@ namespace NKikimr {
TStringStream serialized;
- {
- TChain chain("vdisk", slotsInChunk);
- PreliminaryAllocate(24, chain, arr);
- FreeChunksScenary(chain, arr, chunks);
- chain.Save(&serialized);
- }
- {
- TChain chain("vdisk", slotsInChunk);
- TStringInput str(serialized.Str());
- chain.Load(&str);
- }
+ TChain chain("vdisk", slotsInChunk, 1);
+ PreliminaryAllocate(24, chain, arr);
+ FreeChunksScenary(chain, arr, chunks);
+ chain.Save(&serialized);
+
+ TStringInput str(serialized.Str());
+ bool comp = true;
+ TChain chain2 = TChain::Load(&str, "vdisk", 1 /*appendBlockSize*/, slotsInChunk, {&chain, &chain + 1}, &comp);
}
Y_UNIT_TEST(AllocFreeAllocTest) {
@@ -248,38 +245,6 @@ namespace NKikimr {
}
}
- Y_UNIT_TEST_SUITE(TBlobStorageHullHugeLayout) {
-
- Y_UNIT_TEST(TestOldAppendBlockSize) {
- TAllChains all("vdisk", 134274560, 56896, 512 << 10, 512 << 10, 512 << 10, 10 << 20, 8);
- all.PrintOutChains(STR);
- all.PrintOutSearchTable(STR);
- std::pair<ui32, ui32> p = all.GetTablesSize();
- TVector<NPrivate::TChainLayoutBuilder::TSeg> canonical = {
- {9, 10}, {10, 11}, {11, 12}, {12, 13}, {13, 14}, {14, 15}, {15, 16}, {16, 18}, {18, 20},
- {20, 22}, {22, 24}, {24, 27}, {27, 30}, {30, 33}, {33, 37}, {37, 41}, {41, 46}, {46, 51},
- {51, 57}, {57, 64}, {64, 72}, {72, 81}, {81, 91}, {91, 102}, {102, 114}, {114, 128},
- {128, 144}, {144, 162}, {162, 182}, {182, 204}
- };
- UNIT_ASSERT_EQUAL(all.GetLayout(), canonical);
- UNIT_ASSERT_EQUAL(p, (std::pair<ui32, ui32>(30, 186)));
- }
-
- Y_UNIT_TEST(TestNewAppendBlockSize) {
- TAllChains all("vdisk", 134274560, 4064, 512 << 10, 512 << 10, 512 << 10, 10 << 20, 8);
- all.PrintOutChains(STR);
- all.PrintOutSearchTable(STR);
- TVector<NPrivate::TChainLayoutBuilder::TSeg> canonical = {
- {129, 145}, {145, 163}, {163, 183}, {183, 205}, {205, 230}, {230, 258}, {258, 290},
- {290, 326}, {326, 366}, {366, 411}, {411, 462}, {462, 519}, {519, 583}, {583, 655},
- {655, 736}, {736, 828}, {828, 931}, {931, 1047}, {1047, 1177}, {1177, 1324},
- {1324, 1489}, {1489, 1675}, {1675, 1884}, {1884, 2119}, {2119, 2383}, {2383, 2680}
- };
- UNIT_ASSERT_EQUAL(all.GetLayout(), canonical);
- }
- }
-
-
Y_UNIT_TEST_SUITE(TBlobStorageHullHugeHeap) {
Y_UNIT_TEST(AllocateAllFromOneChunk) {
@@ -290,8 +255,9 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
+ heap.FinishRecovery();
ui32 hugeBlobSize = 6u << 20u;
heap.AddChunk(5);
@@ -314,9 +280,10 @@ namespace NKikimr {
// just serialize/deserialize
TString serialized = heap.Serialize();
- THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
newHeap.ParseFromString(serialized);
+ newHeap.FinishRecovery();
}
void AllocateScenary(THeap &heap, ui32 hugeBlobSize, TVector<THugeSlot> &arr) {
@@ -357,8 +324,9 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
+ heap.FinishRecovery();
TVector<THugeSlot> arr;
AllocateScenary(heap, 6u << 20u, arr);
@@ -372,16 +340,21 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
+ heap.FinishRecovery();
TVector<THugeSlot> arr;
AllocateScenary(heap, 6u << 20u, arr);
+ TString heap1 = heap.ToString();
TString serialized = heap.Serialize();
UNIT_ASSERT(THeap::CheckEntryPoint(serialized));
- THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
newHeap.ParseFromString(serialized);
+ newHeap.FinishRecovery();
+ TString heap2 = newHeap.ToString();
+ UNIT_ASSERT_VALUES_EQUAL(heap1, heap2);
FreeScenary(newHeap, arr);
}
@@ -392,8 +365,9 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
+ heap.FinishRecovery();
heap.RecoveryModeAddChunk(2);
heap.RecoveryModeAddChunk(34);
@@ -418,17 +392,17 @@ namespace NKikimr {
ui32 chunkSize = 134274560u;
ui32 appendBlockSize = 56896u;
ui32 minHugeBlobInBytes = appendBlockSize;
- ui32 minREALHugeBlobInBytes = minHugeBlobInBytes / appendBlockSize * appendBlockSize + 1;
ui32 maxBlobInBytes = MaxVDiskBlobSize;
ui32 overhead = 8u;
ui32 freeChunksReservation = 1;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
+ heap.FinishRecovery();
THugeSlot hugeSlot;
ui32 slotSize;
bool res = false;
- res = heap.Allocate(minREALHugeBlobInBytes, &hugeSlot, &slotSize);
+ res = heap.Allocate(minHugeBlobInBytes, &hugeSlot, &slotSize);
UNIT_ASSERT_EQUAL(res, false); // no chunks
res = heap.Allocate(maxBlobInBytes, &hugeSlot, &slotSize);
UNIT_ASSERT_EQUAL(res, false); // no chunks
@@ -439,52 +413,34 @@ namespace NKikimr {
RollbackFrom_New_To_Old,
};
- void Write_SaveEntryPoint_Restart(EWrite_SaveEntryPoint_Restart mode) {
+ Y_UNIT_TEST(WriteRestore) {
ui32 chunkSize = 134274560u;
ui32 appendBlockSize = 4064u;
ui32 minHugeBlobInBytes = appendBlockSize;
- ui32 oldMinHugeBlobInBytes = 64u << 10u;
ui32 mileStoneBlobInBytes = 512u << 10u;
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- ui32 fromMin = 0;
- ui32 toMin = 0;
- switch (mode) {
- case EWrite_SaveEntryPoint_Restart::MigrateFrom_Old_To_New:
- fromMin = oldMinHugeBlobInBytes;
- toMin = minHugeBlobInBytes;
- break;
- case EWrite_SaveEntryPoint_Restart::RollbackFrom_New_To_Old:
- fromMin = minHugeBlobInBytes;
- toMin = oldMinHugeBlobInBytes;
- break;
- }
- THeap oldHeap("vdisk", chunkSize, appendBlockSize, oldMinHugeBlobInBytes, oldMinHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap oldHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
+ oldHeap.FinishRecovery();
- THeap fromHeap("vdisk", chunkSize, appendBlockSize, fromMin, oldMinHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap fromHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
fromHeap.ParseFromString(oldHeap.Serialize());
+ fromHeap.FinishRecovery();
TVector<THugeSlot> arr;
AllocateScenary(fromHeap, 6u << 20u, arr);
TString serialized = fromHeap.Serialize();
UNIT_ASSERT(THeap::CheckEntryPoint(serialized));
- THeap toHeap("vdisk", chunkSize, appendBlockSize, toMin, oldMinHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap toHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
toHeap.ParseFromString(serialized);
+ toHeap.FinishRecovery();
FreeScenary(toHeap, arr);
}
-
- Y_UNIT_TEST(MigrateFrom_Old_To_New) {
- Write_SaveEntryPoint_Restart(EWrite_SaveEntryPoint_Restart::MigrateFrom_Old_To_New);
- }
-
- Y_UNIT_TEST(RollbackFrom_New_To_Old) {
- Write_SaveEntryPoint_Restart(EWrite_SaveEntryPoint_Restart::RollbackFrom_New_To_Old);
- }
}
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp
index 898271ef6d..f5aa9d561f 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp
@@ -64,7 +64,6 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
- const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -73,10 +72,11 @@ namespace NKikimr {
: VCtx(std::move(vctx))
, LogPos(THullHugeRecoveryLogPos::Default())
, Heap(new NHuge::THeap(VCtx->VDiskLogPrefix, chunkSize, appendBlockSize,
- minHugeBlobInBytes, oldMinHugeBlobInBytes, milestoneHugeBlobInBytes,
+ minHugeBlobInBytes, milestoneHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation))
, Guid(TAppData::RandomProvider->GenRand64())
{
+ Heap->FinishRecovery();
logFunc(VDISKP(VCtx->VDiskLogPrefix,
"Recovery started (guid# %" PRIu64 " entryLsn# null): State# %s",
Guid, ToString().data()));
@@ -86,7 +86,6 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
- const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -97,12 +96,13 @@ namespace NKikimr {
: VCtx(std::move(vctx))
, LogPos(THullHugeRecoveryLogPos::Default())
, Heap(new NHuge::THeap(VCtx->VDiskLogPrefix, chunkSize, appendBlockSize,
- minHugeBlobInBytes, oldMinHugeBlobInBytes, milestoneHugeBlobInBytes,
+ minHugeBlobInBytes, milestoneHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation))
, Guid(TAppData::RandomProvider->GenRand64())
, PersistentLsn(entryPointLsn)
{
ParseFromString(entryPointData);
+ Heap->FinishRecovery();
Y_ABORT_UNLESS(entryPointLsn == LogPos.EntryPointLsn);
logFunc(VDISKP(VCtx->VDiskLogPrefix,
"Recovery started (guid# %" PRIu64 " entryLsn# %" PRIu64 "): State# %s",
@@ -113,7 +113,6 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
- const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -124,12 +123,13 @@ namespace NKikimr {
: VCtx(std::move(vctx))
, LogPos(THullHugeRecoveryLogPos::Default())
, Heap(new NHuge::THeap(VCtx->VDiskLogPrefix, chunkSize, appendBlockSize,
- minHugeBlobInBytes, oldMinHugeBlobInBytes, milestoneHugeBlobInBytes,
+ minHugeBlobInBytes, milestoneHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation))
, Guid(TAppData::RandomProvider->GenRand64())
, PersistentLsn(entryPointLsn)
{
ParseFromArray(entryPointData.GetData(), entryPointData.GetSize());
+ Heap->FinishRecovery();
Y_ABORT_UNLESS(entryPointLsn == LogPos.EntryPointLsn);
logFunc(VDISKP(VCtx->VDiskLogPrefix,
"Recovery started (guid# %" PRIu64 " entryLsn# %" PRIu64 "): State# %s",
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h
index ffac5bdb0c..cea53fbe31 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h
@@ -92,7 +92,6 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
- const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -102,7 +101,6 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
- const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -114,7 +112,6 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
- const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -133,7 +130,7 @@ namespace NKikimr {
static bool CheckEntryPoint(TContiguousSpan data);
TString ToString() const;
void RenderHtml(IOutputStream &str) const;
- ui32 GetMinREALHugeBlobInBytes() const;
+ ui32 GetMinHugeBlobInBytes() const;
ui64 FirstLsnToKeep(ui64 minInFlightLsn = Max<ui64>()) const;
TString FirstLsnToKeepDecomposed() const;
bool WouldNewEntryPointAdvanceLog(ui64 freeUpToLsn, ui64 minInFlightLsn, ui32 itemsAfterCommit) const;
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp
index 8268554c68..cb8bff996e 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp
@@ -38,13 +38,13 @@ namespace NKikimr {
TDelayedResponses DelayedResponses;
bool AllowGarbageCollection = false;
THugeBlobCtxPtr HugeBlobCtx;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
TFields(TIntrusivePtr<THullDs> hullDs,
TIntrusivePtr<TLsnMngr> &&lsnMngr,
TPDiskCtxPtr &&pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
const TActorId skeletonId,
bool runHandoff,
TActorSystem *as,
@@ -60,7 +60,7 @@ namespace NKikimr {
, ActorSystem(as)
, BarrierValidation(barrierValidation)
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
{}
void CutRecoveryLog(const TActorContext &ctx, std::unique_ptr<NPDisk::TEvCutLog> msg) {
@@ -83,7 +83,7 @@ namespace NKikimr {
TIntrusivePtr<TLsnMngr> lsnMngr,
TPDiskCtxPtr pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
const TActorId skeletonId,
bool runHandoff,
THullDbRecovery &&uncond,
@@ -92,7 +92,7 @@ namespace NKikimr {
TActorId hugeKeeperId)
: THullDbRecovery(std::move(uncond))
, Fields(std::make_unique<TFields>(HullDs, std::move(lsnMngr), std::move(pdiskCtx), std::move(hugeBlobCtx),
- minREALHugeBlobInBytes, skeletonId, runHandoff, as, barrierValidation, hugeKeeperId))
+ minHugeBlobInBytes, skeletonId, runHandoff, as, barrierValidation, hugeKeeperId))
{}
THull::~THull() = default;
@@ -130,7 +130,7 @@ namespace NKikimr {
Fields->SetLogNotifierActorId(logNotifierAid);
// actor for LogoBlobs DB
HullDs->LogoBlobs->LIActor = ctx.RegisterWithSameMailbox(CreateLogoBlobsActor(config, HullDs, hullLogCtx,
- Fields->HugeBlobCtx, Fields->MinREALHugeBlobInBytes, loggerId, Fields->LogoBlobsRunTimeCtx, syncLogFirstLsnToKeep));
+ Fields->HugeBlobCtx, Fields->MinHugeBlobInBytes, loggerId, Fields->LogoBlobsRunTimeCtx, syncLogFirstLsnToKeep));
activeActors.Insert(HullDs->LogoBlobs->LIActor, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
// actor for Blocks DB
HullDs->Blocks->LIActor = ctx.RegisterWithSameMailbox(CreateBlocksActor(config, HullDs, hullLogCtx, loggerId,
@@ -711,14 +711,14 @@ namespace NKikimr {
ctx.Send(HullDs->Barriers->LIActor, new TEvPermitGarbageCollection);
}
- void THull::ApplyHugeBlobSize(ui32 minREALHugeBlobInBytes, const TActorContext& ctx) {
- Fields->MinREALHugeBlobInBytes = minREALHugeBlobInBytes;
- ctx.Send(HullDs->LogoBlobs->LIActor, new TEvMinHugeBlobSizeUpdate(minREALHugeBlobInBytes));
+ void THull::ApplyHugeBlobSize(ui32 minHugeBlobInBytes, const TActorContext& ctx) {
+ Fields->MinHugeBlobInBytes = minHugeBlobInBytes;
+ ctx.Send(HullDs->LogoBlobs->LIActor, new TEvMinHugeBlobSizeUpdate(minHugeBlobInBytes));
}
void THull::CompactFreshLogoBlobsIfRequired(const TActorContext& ctx) {
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->HugeBlobCtx,
- Fields->MinREALHugeBlobInBytes, Fields->LogoBlobsRunTimeCtx, ctx, false, Fields->AllowGarbageCollection);
+ Fields->MinHugeBlobInBytes, Fields->LogoBlobsRunTimeCtx, ctx, false, Fields->AllowGarbageCollection);
}
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h
index 9d43370da5..0e1b8cd98a 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h
@@ -62,7 +62,7 @@ namespace NKikimr {
TIntrusivePtr<TLsnMngr> lsnMngr,
TPDiskCtxPtr pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
const TActorId skeletonId,
bool runHandoff,
THullDbRecovery &&uncond,
@@ -215,7 +215,7 @@ namespace NKikimr {
void PermitGarbageCollection(const TActorContext& ctx);
- void ApplyHugeBlobSize(ui32 minREALHugeBlobInBytes, const TActorContext& ctx);
+ void ApplyHugeBlobSize(ui32 minHugeBlobInBytes, const TActorContext& ctx);
void CompactFreshLogoBlobsIfRequired(const TActorContext& ctx);
};
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp
index 1edb7cc955..9df6e15b26 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp
@@ -78,7 +78,7 @@ namespace NKikimr {
void CompactFreshSegment(
TIntrusivePtr<THullDs> &hullDs,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
std::shared_ptr<TLevelIndexRunTimeCtx<TKey, TMemRec>> &rtCtx,
const TActorContext &ctx,
bool allowGarbageCollection)
@@ -107,7 +107,7 @@ namespace NKikimr {
ui64 firstLsn = freshSegment->GetFirstLsn();
ui64 lastLsn = freshSegment->GetLastLsn();
std::unique_ptr<TFreshCompaction> compaction(new TFreshCompaction(
- hullCtx, rtCtx, std::move(hugeBlobCtx), minREALHugeBlobInBytes, freshSegment, freshSegmentSnap,
+ hullCtx, rtCtx, std::move(hugeBlobCtx), minHugeBlobInBytes, freshSegment, freshSegmentSnap,
std::move(barriersSnap), std::move(levelSnap), mergeElementsApproximation, it, firstLsn, lastLsn,
TDuration::Max(), {}, allowGarbageCollection));
@@ -176,7 +176,7 @@ namespace NKikimr {
TInstant NextCompactionWakeup;
bool AllowGarbageCollection = false;
THugeBlobCtxPtr HugeBlobCtx;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
friend class TActorBootstrapped<TThis>;
@@ -233,7 +233,7 @@ namespace NKikimr {
void ScheduleCompaction(const TActorContext &ctx) {
// schedule fresh if required
- CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinREALHugeBlobInBytes, RTCtx, ctx,
+ CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinHugeBlobInBytes, RTCtx, ctx,
FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
if (!Config->BaseInfo.ReadOnly && !RunLevelCompactionSelector(ctx)) {
ScheduleCompactionWakeup(ctx);
@@ -261,7 +261,7 @@ namespace NKikimr {
it.SeekToFirst();
std::unique_ptr<TLevelCompaction> compaction(new TLevelCompaction(HullDs->HullCtx, RTCtx, HugeBlobCtx,
- MinREALHugeBlobInBytes, nullptr, nullptr, std::move(barriersSnap), std::move(levelSnap),
+ MinHugeBlobInBytes, nullptr, nullptr, std::move(barriersSnap), std::move(levelSnap),
mergeElementsApproximation, it, firstLsn, lastLsn, TDuration::Minutes(2), {}, AllowGarbageCollection));
NActors::TActorId actorId = RunInBatchPool(ctx, compaction.release());
ActiveActors.Insert(actorId, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
@@ -571,7 +571,7 @@ namespace NKikimr {
if (FullCompactionState.Enabled()) {
ScheduleCompaction(ctx);
} else {
- CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinREALHugeBlobInBytes, RTCtx,
+ CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinHugeBlobInBytes, RTCtx,
ctx, FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
}
break;
@@ -596,7 +596,7 @@ namespace NKikimr {
RTCtx->SetFreeUpToLsn(freeUpToLsn);
// we check if we need to start fresh compaction, FreeUpToLsn influence our decision
const bool freshCompStarted = CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx,
- MinREALHugeBlobInBytes, RTCtx, ctx, FullCompactionState.ForceFreshCompaction(RTCtx),
+ MinHugeBlobInBytes, RTCtx, ctx, FullCompactionState.ForceFreshCompaction(RTCtx),
AllowGarbageCollection);
// just for valid info output to the log
bool moveEntryPointStarted = false;
@@ -655,7 +655,7 @@ namespace NKikimr {
case E::FRESH_ONLY:
Y_ABORT_UNLESS(FreshOnlyCompactQ.empty() || FreshOnlyCompactQ.back().first <= confirmedLsn);
FreshOnlyCompactQ.emplace_back(confirmedLsn, ev);
- CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinREALHugeBlobInBytes, RTCtx, ctx,
+ CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinHugeBlobInBytes, RTCtx, ctx,
true, AllowGarbageCollection); // ask for forced fresh compaction
break;
}
@@ -686,7 +686,7 @@ namespace NKikimr {
}
void Handle(TEvMinHugeBlobSizeUpdate::TPtr ev, const TActorContext& /*ctx*/) {
- MinREALHugeBlobInBytes = ev->Get()->MinREALHugeBlobInBytes;
+ MinHugeBlobInBytes = ev->Get()->MinHugeBlobInBytes;
}
STRICT_STFUNC(StateFunc,
@@ -714,7 +714,7 @@ namespace NKikimr {
TIntrusivePtr<THullDs> hullDs,
std::shared_ptr<THullLogCtx> hullLogCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TActorId loggerId,
std::shared_ptr<TRunTimeCtx> rtCtx,
std::shared_ptr<NSyncLog::TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep)
@@ -738,7 +738,7 @@ namespace NKikimr {
, ActiveActors(RTCtx->LevelIndex->ActorCtx->ActiveActors)
, LevelStat(HullDs->HullCtx->VCtx->VDiskCounters)
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
{}
};
@@ -747,12 +747,12 @@ namespace NKikimr {
TIntrusivePtr<THullDs> hullDs,
std::shared_ptr<THullLogCtx> hullLogCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TActorId loggerId,
std::shared_ptr<TLevelIndexRunTimeCtx<TKeyLogoBlob, TMemRecLogoBlob>> rtCtx,
std::shared_ptr<NSyncLog::TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep) {
return new TLevelIndexActor<TKeyLogoBlob, TMemRecLogoBlob>(config, hullDs, hullLogCtx, std::move(hugeBlobCtx),
- minREALHugeBlobInBytes, loggerId, rtCtx,syncLogFirstLsnToKeep);
+ minHugeBlobInBytes, loggerId, rtCtx,syncLogFirstLsnToKeep);
}
NActors::IActor* CreateBlocksActor(
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h
index e740a9f47e..c47fe75616 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h
@@ -84,7 +84,7 @@ namespace NKikimr {
void CompactFreshSegment(
TIntrusivePtr<THullDs> &hullDs,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
std::shared_ptr<TLevelIndexRunTimeCtx<TKey, TMemRec>> &rtCtx,
const TActorContext &ctx,
bool allowGarbageCollection);
@@ -93,7 +93,7 @@ namespace NKikimr {
bool CompactFreshSegmentIfRequired(
TIntrusivePtr<THullDs> &hullDs,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
std::shared_ptr<TLevelIndexRunTimeCtx<TKey, TMemRec>> &rtCtx,
const TActorContext &ctx,
bool force,
@@ -102,7 +102,7 @@ namespace NKikimr {
ui64 yardFreeUpToLsn = rtCtx->GetFreeUpToLsn();
bool compact = hullDs->HullCtx->FreshCompaction && rtCtx->LevelIndex->NeedsFreshCompaction(yardFreeUpToLsn, force);
if (compact) {
- CompactFreshSegment<TKey, TMemRec>(hullDs, std::move(hugeBlobCtx), minREALHugeBlobInBytes, rtCtx, ctx,
+ CompactFreshSegment<TKey, TMemRec>(hullDs, std::move(hugeBlobCtx), minHugeBlobInBytes, rtCtx, ctx,
allowGarbageCollection);
}
return compact;
@@ -116,7 +116,7 @@ namespace NKikimr {
TIntrusivePtr<THullDs> hullDs,
std::shared_ptr<THullLogCtx> hullLogCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TActorId loggerId,
std::shared_ptr<TLevelIndexRunTimeCtx<TKeyLogoBlob, TMemRecLogoBlob>> rtCtx,
std::shared_ptr<NSyncLog::TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep);
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h
index 73ffcdc8c3..67c50873f7 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h
@@ -323,7 +323,7 @@ namespace NKikimr {
THullCompaction(THullCtxPtr hullCtx,
const std::shared_ptr<TLevelIndexRunTimeCtx> &rtCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TIntrusivePtr<TFreshSegment> freshSegment,
std::shared_ptr<TFreshSegmentSnapshot> freshSegmentSnap,
TBarriersSnapshot &&barriersSnap,
@@ -346,7 +346,7 @@ namespace NKikimr {
, Hmp(CreateHandoffMap<TKey, TMemRec>(HullCtx, rtCtx->RunHandoff, rtCtx->SkeletonId))
, Gcmp(CreateGcMap<TKey, TMemRec>(HullCtx, mergeElementsApproximation, allowGarbageCollection))
, It(it)
- , Worker(HullCtx, PDiskCtx, std::move(hugeBlobCtx), minREALHugeBlobInBytes, rtCtx->LevelIndex, it,
+ , Worker(HullCtx, PDiskCtx, std::move(hugeBlobCtx), minHugeBlobInBytes, rtCtx->LevelIndex, it,
static_cast<bool>(FreshSegment), firstLsn, lastLsn, restoreDeadline, partitionKey)
, CompactionID(TAppData::RandomProvider->GenRand64())
, SkeletonId(rtCtx->SkeletonId)
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h
index 6fa069c703..57a478a1a2 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h
@@ -116,7 +116,7 @@ namespace NKikimr {
THullCtxPtr HullCtx;
TPDiskCtxPtr PDiskCtx;
THugeBlobCtxPtr HugeBlobCtx;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
// Group Type
const TBlobStorageGroupType GType;
@@ -290,7 +290,7 @@ namespace NKikimr {
THullCompactionWorker(THullCtxPtr hullCtx,
TPDiskCtxPtr pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TIntrusivePtr<TLevelIndex> levelIndex,
const TIterator& it,
bool isFresh,
@@ -301,7 +301,7 @@ namespace NKikimr {
: HullCtx(std::move(hullCtx))
, PDiskCtx(std::move(pdiskCtx))
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
, GType(HullCtx->VCtx->Top->GType)
, LevelIndex(std::move(levelIndex))
, FirstLsn(firstLsn)
@@ -570,7 +570,7 @@ namespace NKikimr {
bool PreprocessItem() {
// finish merging data for this item
if constexpr (LogoBlobs) {
- IndexMerger.Finish(HugeBlobCtx->IsHugeBlob(GType, Key.LogoBlobID(), MinREALHugeBlobInBytes));
+ IndexMerger.Finish(HugeBlobCtx->IsHugeBlob(GType, Key.LogoBlobID(), MinHugeBlobInBytes));
} else {
IndexMerger.Finish(false);
}
diff --git a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
index cda349b521..c71d6c3524 100644
--- a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
+++ b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
@@ -444,7 +444,6 @@ namespace NKikimr {
LocRecCtx->PDiskCtx->Dsk->ChunkSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
- Config->OldMinHugeBlobInBytes,
Config->MilestoneHugeBlobInBytes,
Config->MaxLogoBlobDataSize,
Config->HugeBlobOverhead,
@@ -466,7 +465,6 @@ namespace NKikimr {
LocRecCtx->PDiskCtx->Dsk->ChunkSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
- Config->OldMinHugeBlobInBytes,
Config->MilestoneHugeBlobInBytes,
Config->MaxLogoBlobDataSize,
Config->HugeBlobOverhead,
diff --git a/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp b/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp
index 7a5522c8c9..daf51e2ec3 100644
--- a/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp
@@ -32,7 +32,7 @@ namespace NKikimr {
costSettings.SetWriteSpeedBps(Max<ui64>());
costSettings.SetReadBlockSize(Max<ui64>());
costSettings.SetWriteBlockSize(Max<ui64>());
- costSettings.SetMinREALHugeBlobInBytes(Max<ui32>());
+ costSettings.SetMinHugeBlobInBytes(Max<ui32>());
return costSettings;
}
diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp b/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp
index 41bc8c156d..a9aef05686 100644
--- a/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp
+++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp
@@ -156,7 +156,7 @@ namespace NKikimr {
};
std::shared_ptr<TReplCtx> ReplCtx;
- ui32 NextMinREALHugeBlobInBytes;
+ ui32 NextMinHugeBlobInBytes;
THistory History;
EState State;
TInstant LastReplStart;
@@ -257,7 +257,7 @@ namespace NKikimr {
}
void Handle(TEvMinHugeBlobSizeUpdate::TPtr ev) {
- NextMinREALHugeBlobInBytes = ev->Get()->MinREALHugeBlobInBytes;
+ NextMinHugeBlobInBytes = ev->Get()->MinHugeBlobInBytes;
}
void StartReplication() {
@@ -270,8 +270,8 @@ namespace NKikimr {
ReplCtx->MonGroup.ReplWorkUnitsDone() = 0;
ReplCtx->MonGroup.ReplItemsRemaining() = 0;
ReplCtx->MonGroup.ReplItemsDone() = 0;
- Y_ABORT_UNLESS(NextMinREALHugeBlobInBytes);
- ReplCtx->MinREALHugeBlobInBytes = NextMinREALHugeBlobInBytes;
+ Y_ABORT_UNLESS(NextMinHugeBlobInBytes);
+ ReplCtx->MinHugeBlobInBytes = NextMinHugeBlobInBytes;
UnrecoveredNonphantomBlobs = false;
Become(&TThis::StateRepl);
@@ -697,7 +697,7 @@ namespace NKikimr {
TReplScheduler(std::shared_ptr<TReplCtx> &replCtx)
: TActorBootstrapped<TReplScheduler>()
, ReplCtx(replCtx)
- , NextMinREALHugeBlobInBytes(ReplCtx->MinREALHugeBlobInBytes)
+ , NextMinHugeBlobInBytes(ReplCtx->MinHugeBlobInBytes)
, History(HistorySize)
, State(Relaxation)
, ReplProgressWatchdog(
diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h b/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h
index a8ac714d52..2d2211f63c 100644
--- a/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h
+++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h
@@ -15,7 +15,7 @@ namespace NKikimr {
TIntrusivePtr<THullCtx> HullCtx;
TPDiskCtxPtr PDiskCtx;
std::shared_ptr<THugeBlobCtx> HugeBlobCtx;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
TIntrusivePtr<THullDs> HullDs;
TIntrusivePtr<TBlobStorageGroupInfo> GInfo;
TActorId SkeletonId;
@@ -31,7 +31,7 @@ namespace NKikimr {
TIntrusivePtr<THullCtx> hullCtx,
TPDiskCtxPtr pdiskCtx,
std::shared_ptr<THugeBlobCtx> hugeBlobCtx,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TIntrusivePtr<THullDs> hullDs,
TIntrusivePtr<TBlobStorageGroupInfo> info,
const TActorId &skeletonId,
@@ -42,7 +42,7 @@ namespace NKikimr {
, HullCtx(std::move(hullCtx))
, PDiskCtx(std::move(pdiskCtx))
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
, HullDs(std::move(hullDs))
, GInfo(std::move(info))
, SkeletonId(skeletonId)
@@ -51,7 +51,7 @@ namespace NKikimr {
, PDiskWriteBytes(std::move(pdiskWriteBytes))
, PausedAtStart(pausedAtStart)
{
- Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinHugeBlobInBytes);
}
bool GetAddHeader() const { return !HullCtx || HullCtx->AddHeader; }
diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h b/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h
index 10dcccd3dc..3de2562e82 100644
--- a/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h
+++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h
@@ -206,7 +206,7 @@ namespace NKikimr {
partsSize += partSize;
TRope& data = item.Parts[i];
Y_ABORT_UNLESS(data.GetSize() == partSize);
- if (ReplCtx->HugeBlobCtx->IsHugeBlob(groupType, id, ReplCtx->MinREALHugeBlobInBytes)) {
+ if (ReplCtx->HugeBlobCtx->IsHugeBlob(groupType, id, ReplCtx->MinHugeBlobInBytes)) {
AddBlobToQueue(partId, TDiskBlob::Create(id.BlobSize(), i + 1, groupType.TotalPartCount(),
std::move(data), Arena, ReplCtx->GetAddHeader()), {}, true, rbq);
++numHuge;
@@ -380,7 +380,7 @@ namespace NKikimr {
void RecoverMetadata(const TLogoBlobID& id, TRecoveredBlobsQueue& rbq) {
while (!MetadataParts.empty() && MetadataParts.front().FullID() <= id) {
const TLogoBlobID id = MetadataParts.front();
- const bool isHugeBlob = ReplCtx->HugeBlobCtx->IsHugeBlob(ReplCtx->VCtx->Top->GType, id.FullID(), ReplCtx->MinREALHugeBlobInBytes);
+ const bool isHugeBlob = ReplCtx->HugeBlobCtx->IsHugeBlob(ReplCtx->VCtx->Top->GType, id.FullID(), ReplCtx->MinHugeBlobInBytes);
MetadataParts.pop_front();
STLOG(PRI_DEBUG, BS_REPL, BSVR30, VDISKP(ReplCtx->VCtx->VDiskLogPrefix,
"TRecoveryMachine::RecoverMetadata"), (BlobId, id));
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
index 0832fbc623..def1404392 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
@@ -202,25 +202,25 @@ namespace NKikimr {
if (!ApplyHugeBlobSize(type.GetMinHugeBlobSizeInBytes())) {
continue;
}
- Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinHugeBlobInBytes);
if (Config->RunRepl) {
- ctx.Send(Db->ReplID, new TEvMinHugeBlobSizeUpdate(MinREALHugeBlobInBytes));
+ ctx.Send(Db->ReplID, new TEvMinHugeBlobSizeUpdate(MinHugeBlobInBytes));
}
if (Hull) {
- Hull->ApplyHugeBlobSize(MinREALHugeBlobInBytes, ctx);
+ Hull->ApplyHugeBlobSize(MinHugeBlobInBytes, ctx);
}
- ctx.Send(*SkeletonFrontIDPtr, new TEvMinHugeBlobSizeUpdate(MinREALHugeBlobInBytes));
+ ctx.Send(*SkeletonFrontIDPtr, new TEvMinHugeBlobSizeUpdate(MinHugeBlobInBytes));
}
}
}
bool ApplyHugeBlobSize(ui32 minHugeBlobInBytes) {
- ui32 alignedSize = HugeBlobCtx->HugeSlotsMap->AlignByBlockSize(minHugeBlobInBytes) + 1;
- if (MinREALHugeBlobInBytes == alignedSize) {
+ const ui32 alignedSize = HugeBlobCtx->HugeSlotsMap->AlignByBlockSize(minHugeBlobInBytes);
+ if (MinHugeBlobInBytes == alignedSize) {
return false;
}
- MinREALHugeBlobInBytes = alignedSize;
- IFaceMonGroup->MinHugeBlobInBytes(MinREALHugeBlobInBytes);
+ MinHugeBlobInBytes = alignedSize;
+ IFaceMonGroup->MinHugeBlobInBytes(MinHugeBlobInBytes);
return true;
}
@@ -583,7 +583,7 @@ namespace NKikimr {
TVPutInfo &info = putsInfo.back();
try {
- info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, blobId.FullID(), MinREALHugeBlobInBytes);
+ info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, blobId.FullID(), MinHugeBlobInBytes);
if (info.IsHugeBlob) {
LOG_CRIT_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: TEvVMultiPut has huge blob# "
<< blobId << " Marker# BSVS08");
@@ -726,7 +726,7 @@ namespace NKikimr {
const ui64 bufSize = info.Buffer.GetSize();
try {
- info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, id.FullID(), MinREALHugeBlobInBytes);
+ info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, id.FullID(), MinHugeBlobInBytes);
} catch (yexception ex) {
LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << ex.what() << " Marker# BSVS41");
info.HullStatus = {NKikimrProto::ERROR, "", false};
@@ -1788,7 +1788,7 @@ namespace NKikimr {
auto msg = std::make_unique<TEvFrontRecoveryStatus>(TEvFrontRecoveryStatus::SyncGuidRecoveryDone,
NKikimrProto::OK,
(PDiskCtx ? PDiskCtx->Dsk : nullptr),
- MinREALHugeBlobInBytes,
+ MinHugeBlobInBytes,
Db->GetVDiskIncarnationGuid());
ctx.Send(*SkeletonFrontIDPtr, msg.release());
ctx.Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(
@@ -1815,7 +1815,7 @@ namespace NKikimr {
auto msg = std::make_unique<TEvFrontRecoveryStatus>(phase,
NKikimrProto::ERROR,
(PDiskCtx ? PDiskCtx->Dsk : nullptr),
- MinREALHugeBlobInBytes,
+ MinHugeBlobInBytes,
Db->GetVDiskIncarnationGuid());
ctx.Send(*SkeletonFrontIDPtr, msg.release());
// push the status
@@ -1863,7 +1863,7 @@ namespace NKikimr {
// check status
if (ev->Get()->Status == NKikimrProto::OK) {
ApplyHugeBlobSize(Config->MinHugeBlobInBytes);
- Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinHugeBlobInBytes);
// handle special case when donor disk starts and finds out that it has been wiped out
if (ev->Get()->LsnMngr->GetOriginallyRecoveredLsn() == 0 && Config->BaseInfo.DonorMode) {
@@ -1881,7 +1881,7 @@ namespace NKikimr {
auto msg = std::make_unique<TEvFrontRecoveryStatus>(TEvFrontRecoveryStatus::LocalRecoveryDone,
NKikimrProto::OK,
PDiskCtx->Dsk,
- MinREALHugeBlobInBytes,
+ MinHugeBlobInBytes,
Db->GetVDiskIncarnationGuid());
ctx.Send(*SkeletonFrontIDPtr, msg.release());
@@ -1943,7 +1943,7 @@ namespace NKikimr {
Db->HugeKeeperID);
// create Hull
- Hull = std::make_shared<THull>(Db->LsnMngr, PDiskCtx, HugeBlobCtx, MinREALHugeBlobInBytes,
+ Hull = std::make_shared<THull>(Db->LsnMngr, PDiskCtx, HugeBlobCtx, MinHugeBlobInBytes,
Db->SkeletonID, Config->BalancingEnableDelete, std::move(*ev->Get()->Uncond),
ctx.ExecutorThread.ActorSystem, Config->BarrierValidation, Db->HugeKeeperID);
ActiveActors.Insert(Hull->RunHullServices(Config, HullLogCtx, Db->SyncLogFirstLsnToKeep,
@@ -2044,7 +2044,7 @@ namespace NKikimr {
DbBirthLsn = ev->Get()->DbBirthLsn;
SkeletonIsUpAndRunning(ctx, Config->RunRepl);
if (Config->RunRepl) {
- auto replCtx = std::make_shared<TReplCtx>(VCtx, HullCtx, PDiskCtx, HugeBlobCtx, MinREALHugeBlobInBytes, Hull->GetHullDs(),
+ auto replCtx = std::make_shared<TReplCtx>(VCtx, HullCtx, PDiskCtx, HugeBlobCtx, MinHugeBlobInBytes, Hull->GetHullDs(),
GInfo, SelfId(), Config, PDiskWriteBytes, Config->ReplPausedAtStart);
Db->ReplID.Set(ctx.Register(CreateReplActor(replCtx)));
ActiveActors.Insert(Db->ReplID, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE); // keep forever
@@ -2594,7 +2594,7 @@ namespace NKikimr {
.TimeToSleepIfNothingToDo=Config->BalancingTimeToSleepIfNothingToDo,
};
auto balancingCtx = std::make_shared<TBalancingCtx>(
- balancingCfg, VCtx, PDiskCtx, HugeBlobCtx, SelfId(), Hull->GetSnapshot(), Config, GInfo, MinREALHugeBlobInBytes);
+ balancingCfg, VCtx, PDiskCtx, HugeBlobCtx, SelfId(), Hull->GetSnapshot(), Config, GInfo, MinHugeBlobInBytes);
BalancingId = RunInBatchPool(ctx, CreateBalancingActor(balancingCtx));
ActiveActors.Insert(BalancingId, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
}
@@ -2857,7 +2857,7 @@ namespace NKikimr {
THullCtxPtr HullCtx;
THugeBlobCtxPtr HugeBlobCtx;
std::shared_ptr<THullLogCtx> HullLogCtx;
- ui32 MinREALHugeBlobInBytes = 0;
+ ui32 MinHugeBlobInBytes = 0;
std::shared_ptr<THull> Hull; // run it after local recovery
std::shared_ptr<TOutOfSpaceLogic> OutOfSpaceLogic;
std::shared_ptr<TQueryCtx> QueryCtx;
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
index 9d9b820c54..7d9f49f6cd 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
@@ -42,12 +42,12 @@ namespace NKikimr {
TEvFrontRecoveryStatus::TEvFrontRecoveryStatus(EPhase phase,
NKikimrProto::EReplyStatus status,
const TIntrusivePtr<TPDiskParams> &dsk,
- ui32 minREALHugeBlobInBytes,
+ ui32 minHugeBlobInBytes,
TVDiskIncarnationGuid vdiskIncarnationGuid)
: Phase(phase)
, Status(status)
, Dsk(dsk)
- , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
+ , MinHugeBlobInBytes(minHugeBlobInBytes)
, VDiskIncarnationGuid(vdiskIncarnationGuid)
{}
@@ -816,7 +816,7 @@ namespace NKikimr {
TBlobStorageGroupType type = (GInfo ? GInfo->Type : TErasureType::ErasureNone);
VCtx->UpdateCostModel(std::make_unique<TCostModel>(msg->Dsk->SeekTimeUs, msg->Dsk->ReadSpeedBps,
msg->Dsk->WriteSpeedBps, msg->Dsk->ReadBlockSize, msg->Dsk->WriteBlockSize,
- msg->MinREALHugeBlobInBytes, type));
+ msg->MinHugeBlobInBytes, type));
break;
}
case TEvFrontRecoveryStatus::SyncGuidRecoveryDone:
@@ -831,7 +831,7 @@ namespace NKikimr {
void Handle(TEvMinHugeBlobSizeUpdate::TPtr &ev) {
VCtx->UpdateCostModel(std::make_unique<TCostModel>(VCtx->CostModel->SeekTimeUs, VCtx->CostModel->ReadSpeedBps,
VCtx->CostModel->WriteSpeedBps, VCtx->CostModel->ReadBlockSize, VCtx->CostModel->WriteBlockSize,
- ev->Get()->MinREALHugeBlobInBytes, VCtx->CostModel->GType));
+ ev->Get()->MinHugeBlobInBytes, VCtx->CostModel->GType));
}
static NKikimrWhiteboard::EFlag ToLightSignal(NKikimrWhiteboard::EVDiskState st) {
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h
index d5a7c13ad6..068edefb02 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h
@@ -24,13 +24,13 @@ namespace NKikimr {
const EPhase Phase;
const NKikimrProto::EReplyStatus Status;
const TIntrusivePtr<TPDiskParams> Dsk;
- ui32 MinREALHugeBlobInBytes;
+ ui32 MinHugeBlobInBytes;
const TVDiskIncarnationGuid VDiskIncarnationGuid;
TEvFrontRecoveryStatus(EPhase phase,
NKikimrProto::EReplyStatus status,
const TIntrusivePtr<TPDiskParams> &dsk,
- ui32 MinREALHugeBlobInBytes,
+ ui32 MinHugeBlobInBytes,
TVDiskIncarnationGuid vdiskIncarnationGuid);
~TEvFrontRecoveryStatus();
};
diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
index 3c4aa292d2..554ca1febb 100644
--- a/ydb/core/protos/blobstorage.proto
+++ b/ydb/core/protos/blobstorage.proto
@@ -25,7 +25,7 @@ message TVDiskCostSettings {
optional uint64 WriteSpeedBps = 3;
optional uint64 ReadBlockSize = 4;
optional uint64 WriteBlockSize = 5;
- optional uint32 MinREALHugeBlobInBytes = 6;
+ optional uint32 MinHugeBlobInBytes = 6;
};
message TWindowFeedback {