aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander Rutkovsky <alexvru@ydb.tech>2024-11-22 23:12:15 +0300
committerGitHub <noreply@github.com>2024-11-22 21:12:15 +0100
commitf5beef5e566eab65072c276a69a3465656c521d4 (patch)
tree4ee747fb8f08fdf33907bfc444d70d6c982e5929
parent663d308482a833d447764365d47135e4ecac3d58 (diff)
downloadydb-f5beef5e566eab65072c276a69a3465656c521d4.tar.gz
Revert "Support arbitrary chain set in huge blob keeper heap (#11566)" (#11891)
-rw-r--r--ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_impl.h2
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_request.cpp5
-rw-r--r--ydb/core/blobstorage/dsproxy/dsproxy_state.cpp4
-rw-r--r--ydb/core/blobstorage/dsproxy/group_sessions.cpp4
-rw-r--r--ydb/core/blobstorage/dsproxy/group_sessions.h2
-rw-r--r--ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp10
-rw-r--r--ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp3
-rw-r--r--ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/balance/defs.h6
-rw-r--r--ydb/core/blobstorage/vdisk/balance/sender.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_config.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_config.h1
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp18
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h6
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_events.h8
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp8
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h2
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp26
-rw-r--r--ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h12
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp5
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp609
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h137
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp1
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp114
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp12
-rw-r--r--ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h5
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp20
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h4
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp26
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h8
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h4
-rw-r--r--ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h8
-rw-r--r--ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp2
-rw-r--r--ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp10
-rw-r--r--ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h8
-rw-r--r--ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h4
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp36
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp8
-rw-r--r--ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h4
-rw-r--r--ydb/core/protos/blobstorage.proto2
44 files changed, 637 insertions, 521 deletions
diff --git a/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h b/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h
index 9518ad7ced..66805e3a1d 100644
--- a/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h
+++ b/ydb/core/blobstorage/backpressure/ut_client/skeleton_front_mock.h
@@ -150,7 +150,7 @@ public:
settings->SetWriteSpeedBps(60e6);
settings->SetReadBlockSize(4096);
settings->SetWriteBlockSize(4096);
- settings->SetMinHugeBlobInBytes(512 << 10);
+ settings->SetMinREALHugeBlobInBytes(512 << 10);
}
STRICT_STFUNC(StateFunc, {
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy.h b/ydb/core/blobstorage/dsproxy/dsproxy.h
index e0ae85c22d..d8f06ae9f0 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy.h
+++ b/ydb/core/blobstorage/dsproxy/dsproxy.h
@@ -30,7 +30,7 @@ LWTRACE_USING(BLOBSTORAGE_PROVIDER);
constexpr ui32 TypicalPartsInBlob = 6;
constexpr ui32 TypicalDisksInSubring = 8;
-constexpr ui32 MaxBatchedPutSize = 64 * 1024 - 512 - 5; // (MinHugeBlobInBytes - 1 - TDiskBlob::HugeBlobOverhead) for ssd and nvme
+constexpr ui32 MaxBatchedPutSize = 64 * 1024 - 512 - 5; // (MinREALHugeBlobInBytes - 1 - TDiskBlob::HugeBlobOverhead) for ssd and nvme
const TDuration ProxyConfigurationTimeout = TDuration::Seconds(20);
const ui32 ProxyRetryConfigurationInitialTimeout = 200;
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_impl.h b/ydb/core/blobstorage/dsproxy/dsproxy_impl.h
index 3d7fb25129..dfdbe32b70 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_impl.h
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_impl.h
@@ -71,7 +71,7 @@ class TBlobStorageGroupProxy : public TActorBootstrapped<TBlobStorageGroupProxy>
bool UseActorSystemTimeInBSQueue;
bool IsLimitedKeyless = false;
bool IsFullMonitoring = false; // current state of monitoring
- ui32 MinHugeBlobInBytes = 0;
+ ui32 MinREALHugeBlobInBytes = 0;
TActorId MonActor;
TIntrusivePtr<TBlobStorageGroupProxyMon> Mon;
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
index 4ffe2a10e2..4575b666b8 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_request.cpp
@@ -178,12 +178,13 @@ namespace NKikimr {
Send(MonActor, new TEvThroughputAddRequest(ev->Get()->HandleClass, bytes));
EnableWilsonTracing(ev, Mon->PutSamplePPM);
- Y_DEBUG_ABORT_UNLESS(MinHugeBlobInBytes);
+ Y_DEBUG_ABORT_UNLESS(MinREALHugeBlobInBytes);
const ui32 partSize = Info->Type.PartSize(ev->Get()->Id);
TInstant now = TActivationContext::Now();
- if (Controls.EnablePutBatching.Update(now) && partSize < MinHugeBlobInBytes && partSize <= MaxBatchedPutSize) {
+ if (Controls.EnablePutBatching.Update(now) && partSize < MinREALHugeBlobInBytes &&
+ partSize <= MaxBatchedPutSize) {
NKikimrBlobStorage::EPutHandleClass handleClass = ev->Get()->HandleClass;
TEvBlobStorage::TEvPut::ETactic tactic = ev->Get()->Tactic;
Y_ABORT_UNLESS((ui64)handleClass <= PutHandleClassCount);
diff --git a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp
index b69f3f6aea..3cb1b3fd44 100644
--- a/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp
+++ b/ydb/core/blobstorage/dsproxy/dsproxy_state.cpp
@@ -208,7 +208,7 @@ namespace NKikimr {
Y_ABORT_UNLESS(Topology);
Sessions->QueueConnectUpdate(Topology->GetOrderNumber(msg->VDiskId), msg->QueueId, msg->IsConnected,
msg->ExtraBlockChecksSupport, msg->CostModel, *Topology);
- MinHugeBlobInBytes = Sessions->GetMinHugeBlobInBytes();
+ MinREALHugeBlobInBytes = Sessions->GetMinREALHugeBlobInBytes();
if (msg->IsConnected && (CurrentStateFunc() == &TThis::StateEstablishingSessions ||
CurrentStateFunc() == &TThis::StateEstablishingSessionsTimeout)) {
SwitchToWorkWhenGoodToGo();
@@ -216,7 +216,7 @@ namespace NKikimr {
SetStateEstablishingSessions();
}
- Y_DEBUG_ABORT_UNLESS(CurrentStateFunc() != &TThis::StateWork || MinHugeBlobInBytes);
+ Y_DEBUG_ABORT_UNLESS(CurrentStateFunc() != &TThis::StateWork || MinREALHugeBlobInBytes);
if (const ui32 prev = std::exchange(NumUnconnectedDisks, Sessions->GetNumUnconnectedDisks()); prev != NumUnconnectedDisks) {
NodeMon->IncNumUnconnected(NumUnconnectedDisks);
diff --git a/ydb/core/blobstorage/dsproxy/group_sessions.cpp b/ydb/core/blobstorage/dsproxy/group_sessions.cpp
index 24b33a06d5..8569a94ad9 100644
--- a/ydb/core/blobstorage/dsproxy/group_sessions.cpp
+++ b/ydb/core/blobstorage/dsproxy/group_sessions.cpp
@@ -188,8 +188,8 @@ void TGroupSessions::QueueConnectUpdate(ui32 orderNumber, NKikimrBlobStorage::EV
}
}
-ui32 TGroupSessions::GetMinHugeBlobInBytes() const {
- return GroupQueues->CostModel ? GroupQueues->CostModel->MinHugeBlobInBytes : 0;
+ui32 TGroupSessions::GetMinREALHugeBlobInBytes() const {
+ return GroupQueues->CostModel ? GroupQueues->CostModel->MinREALHugeBlobInBytes : 0;
}
ui32 TGroupSessions::GetNumUnconnectedDisks() {
diff --git a/ydb/core/blobstorage/dsproxy/group_sessions.h b/ydb/core/blobstorage/dsproxy/group_sessions.h
index 8bb706d4cd..174364976a 100644
--- a/ydb/core/blobstorage/dsproxy/group_sessions.h
+++ b/ydb/core/blobstorage/dsproxy/group_sessions.h
@@ -210,7 +210,7 @@ namespace NKikimr {
void QueueConnectUpdate(ui32 orderNumber, NKikimrBlobStorage::EVDiskQueueId queueId, bool connected,
bool extraBlockChecksSupport, std::shared_ptr<const TCostModel> costModel, const TBlobStorageGroupInfo::TTopology& topology);
ui32 GetNumUnconnectedDisks();
- ui32 GetMinHugeBlobInBytes() const;
+ ui32 GetMinREALHugeBlobInBytes() const;
};
struct TEvRequestProxySessionsState : TEventLocal<TEvRequestProxySessionsState, TEvBlobStorage::EvRequestProxySessionsState>
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp
index 1df9dc3a81..ba0477182a 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/helpers.cpp
@@ -394,7 +394,7 @@ class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> {
bool Started = false;
// how many deadline statuses we got
ui64 RequestDeadlines = 0;
- ui32 MinHugeBlobInBytes = 0;
+ ui32 MinREALHugeBlobInBytes = 0;
ui64 LastBatchSize = 0;
@@ -413,8 +413,8 @@ class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> {
void Handle(TEvProxyQueueState::TPtr& ev, const TActorContext& ctx) {
if (ev->Get()->IsConnected && !Started) {
// put logo blob
- MinHugeBlobInBytes = ev->Get()->CostModel->MinHugeBlobInBytes;
- Y_ABORT_UNLESS(MinHugeBlobInBytes);
+ MinREALHugeBlobInBytes = ev->Get()->CostModel->MinREALHugeBlobInBytes;
+ Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
SendPut(ctx);
Started = true;
}
@@ -485,12 +485,12 @@ class TManyMultiPuts : public TActorBootstrapped<TManyMultiPuts> {
Y_ABORT_UNLESS(status == NKikimrProto::OK || noTimeout && status == NKikimrProto::DEADLINE,
"Event# %s", ev->Get()->ToString().data());
- Y_ABORT_UNLESS(MinHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
switch (status) {
case NKikimrProto::OK:
for (auto &item : record.GetItems()) {
- Y_ABORT_UNLESS(item.GetStatus() == (MsgData.size() < MinHugeBlobInBytes ? NKikimrProto::OK : NKikimrProto::ERROR));
+ Y_ABORT_UNLESS(item.GetStatus() == (MsgData.size() < MinREALHugeBlobInBytes ? NKikimrProto::OK : NKikimrProto::ERROR));
}
break;
case NKikimrProto::DEADLINE:
diff --git a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
index 5fabd73846..ad9b486e67 100644
--- a/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
+++ b/ydb/core/blobstorage/ut_vdisk/lib/test_huge.cpp
@@ -137,6 +137,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
bool InitHugeBlobKeeper(const TActorContext &ctx, const TStartingPoints &startingPoints) {
Y_UNUSED(ctx);
+ const ui32 oldMinHugeBlobInBytes = 64 << 10;
const ui32 milestoneHugeBlobInBytes = 64 << 10;
const ui32 maxBlobInBytes = 128 << 10;
auto logFunc = [] (const TString) { /* empty */ };
@@ -149,6 +150,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
HmCtx->PDiskCtx->Dsk->ChunkSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
+ oldMinHugeBlobInBytes,
milestoneHugeBlobInBytes,
maxBlobInBytes,
HmCtx->Config->HugeBlobOverhead,
@@ -167,6 +169,7 @@ class THugeModuleRecoveryActor : public TActorBootstrapped<THugeModuleRecoveryAc
HmCtx->PDiskCtx->Dsk->ChunkSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
HmCtx->PDiskCtx->Dsk->AppendBlockSize,
+ oldMinHugeBlobInBytes,
milestoneHugeBlobInBytes,
maxBlobInBytes,
HmCtx->Config->HugeBlobOverhead,
diff --git a/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp b/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp
index b0d3585439..0a7b8abd73 100644
--- a/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp
+++ b/ydb/core/blobstorage/vdisk/balance/balancing_actor.cpp
@@ -142,7 +142,7 @@ namespace NBalancing {
const auto& key = It.GetCurKey().LogoBlobID();
- if (Ctx->Cfg.BalanceOnlyHugeBlobs && !Ctx->HugeBlobCtx->IsHugeBlob(GInfo->Type, key, Ctx->MinHugeBlobInBytes)) {
+ if (Ctx->Cfg.BalanceOnlyHugeBlobs && !Ctx->HugeBlobCtx->IsHugeBlob(GInfo->Type, key, Ctx->MinREALHugeBlobInBytes)) {
// skip non huge blobs
continue;
}
diff --git a/ydb/core/blobstorage/vdisk/balance/defs.h b/ydb/core/blobstorage/vdisk/balance/defs.h
index 63f4fecaa9..b94d6a3b2f 100644
--- a/ydb/core/blobstorage/vdisk/balance/defs.h
+++ b/ydb/core/blobstorage/vdisk/balance/defs.h
@@ -42,7 +42,7 @@ namespace NKikimr {
TIntrusivePtr<TVDiskConfig> VDiskCfg;
TIntrusivePtr<TBlobStorageGroupInfo> GInfo;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
TBalancingCtx(
const TBalancingCfg& cfg,
@@ -53,7 +53,7 @@ namespace NKikimr {
NKikimr::THullDsSnap snap,
TIntrusivePtr<TVDiskConfig> vDiskCfg,
TIntrusivePtr<TBlobStorageGroupInfo> gInfo,
- ui32 minHugeBlobInBytes
+ ui32 minREALHugeBlobInBytes
)
: Cfg(cfg)
, VCtx(std::move(vCtx))
@@ -64,7 +64,7 @@ namespace NKikimr {
, Snap(std::move(snap))
, VDiskCfg(std::move(vDiskCfg))
, GInfo(std::move(gInfo))
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
{
}
};
diff --git a/ydb/core/blobstorage/vdisk/balance/sender.cpp b/ydb/core/blobstorage/vdisk/balance/sender.cpp
index a455abb572..0fc8b9083f 100644
--- a/ydb/core/blobstorage/vdisk/balance/sender.cpp
+++ b/ydb/core/blobstorage/vdisk/balance/sender.cpp
@@ -153,7 +153,7 @@ namespace {
auto& data = part.PartsData[i];
auto vDiskId = GetMainReplicaVDiskId(*GInfo, key);
- if (Ctx->HugeBlobCtx->IsHugeBlob(GInfo->GetTopology().GType, part.Key, Ctx->MinHugeBlobInBytes)) {
+ if (Ctx->HugeBlobCtx->IsHugeBlob(GInfo->GetTopology().GType, part.Key, Ctx->MinREALHugeBlobInBytes)) {
auto ev = std::make_unique<TEvBlobStorage::TEvVPut>(
key, data, vDiskId,
true, nullptr,
diff --git a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h
index 3cdb566158..4b5c7283ab 100644
--- a/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h
+++ b/ydb/core/blobstorage/vdisk/common/blobstorage_cost_tracker.h
@@ -93,7 +93,7 @@ public:
DeviceWriteSpeedBps = costModel.WriteSpeedBps;
DeviceReadBlockSize = costModel.ReadBlockSize;
DeviceWriteBlockSize = costModel.WriteBlockSize;
- HugeBlobSize = costModel.MinHugeBlobInBytes;
+ HugeBlobSize = costModel.MinREALHugeBlobInBytes;
}
protected:
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
index edfcd7c452..540edfaac5 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.cpp
@@ -136,7 +136,9 @@ namespace NKikimr {
MinHugeBlobInBytes = 512u << 10u;
break;
}
+ OldMinHugeBlobInBytes = MinHugeBlobInBytes; // preserved to migrate entry point state correctly
MilestoneHugeBlobInBytes = 512u << 10u; // for compatibility reasons it must be 512KB
+
}
void TVDiskConfig::Merge(const NKikimrBlobStorage::TVDiskConfig &update) {
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_config.h b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
index 70bfdd1ee1..dd80ff86ce 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_config.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_config.h
@@ -120,6 +120,7 @@ namespace NKikimr {
ui32 HullSstSizeInChunksLevel;
ui32 HugeBlobsFreeChunkReservation;
ui32 MinHugeBlobInBytes;
+ ui32 OldMinHugeBlobInBytes;
ui32 MilestoneHugeBlobInBytes;
ui32 HugeBlobOverhead;
ui32 HullCompLevel0MaxSstsAtOnce;
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp
index 4d4ed7d0ed..9d78d90e15 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.cpp
@@ -95,13 +95,13 @@ namespace NKikimr {
}
TCostModel::TCostModel(ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize,
- ui64 writeBlockSize, ui32 minHugeBlobInBytes, TBlobStorageGroupType gType)
+ ui64 writeBlockSize, ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType)
: SeekTimeUs(seekTimeUs)
, ReadSpeedBps(readSpeedBps)
, WriteSpeedBps(writeSpeedBps)
, ReadBlockSize(readBlockSize)
, WriteBlockSize(writeBlockSize)
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
, GType(gType)
{}
@@ -111,7 +111,7 @@ namespace NKikimr {
, WriteSpeedBps(settings.GetWriteSpeedBps())
, ReadBlockSize(settings.GetReadBlockSize())
, WriteBlockSize(settings.GetWriteBlockSize())
- , MinHugeBlobInBytes(settings.GetMinHugeBlobInBytes())
+ , MinREALHugeBlobInBytes(settings.GetMinREALHugeBlobInBytes())
, GType(gType)
{}
@@ -121,7 +121,7 @@ namespace NKikimr {
settings.SetWriteSpeedBps(WriteSpeedBps);
settings.SetReadBlockSize(ReadBlockSize);
settings.SetWriteBlockSize(WriteBlockSize);
- settings.SetMinHugeBlobInBytes(MinHugeBlobInBytes);
+ settings.SetMinREALHugeBlobInBytes(MinREALHugeBlobInBytes);
}
/// READS
@@ -181,7 +181,7 @@ namespace NKikimr {
const NKikimrBlobStorage::EPutHandleClass handleClass = record.GetHandleClass();
const ui64 bufSize = record.HasBuffer() ? record.GetBuffer().size() : ev.GetPayload(0).GetSize();
- NPriPut::EHandleType handleType = NPriPut::HandleType(MinHugeBlobInBytes, handleClass, bufSize, true);
+ NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, handleClass, bufSize, true);
if (handleType == NPriPut::Log) {
*logPutInternalQueue = true;
return SmallWriteCost(bufSize);
@@ -198,7 +198,7 @@ namespace NKikimr {
ui64 cost = 0;
for (ui64 idx = 0; idx < record.ItemsSize(); ++idx) {
const ui64 size = ev.GetBufferBytes(idx);
- NPriPut::EHandleType handleType = NPriPut::HandleType(MinHugeBlobInBytes, handleClass, size, true);
+ NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, handleClass, size, true);
if (handleType == NPriPut::Log) {
cost += SmallWriteCost(size);
} else {
@@ -265,7 +265,7 @@ namespace NKikimr {
cost += MovedPatchCostBySize(essence.MovedPatchBlobSize);
}
for (ui64 size : essence.PutBufferSizes) {
- NPriPut::EHandleType handleType = NPriPut::HandleType(MinHugeBlobInBytes, essence.HandleClass, size, true);
+ NPriPut::EHandleType handleType = NPriPut::HandleType(MinREALHugeBlobInBytes, essence.HandleClass, size, true);
if (handleType == NPriPut::Log) {
cost += SmallWriteCost(size);
} else {
@@ -283,7 +283,7 @@ namespace NKikimr {
str << " WriteSpeedBps# " << WriteSpeedBps;
str << " ReadBlockSize# " << ReadBlockSize;
str << " WriteBlockSize# " << WriteBlockSize;
- str << " MinHugeBlobInBytes# " << MinHugeBlobInBytes;
+ str << " MinREALHugeBlobInBytes# " << MinREALHugeBlobInBytes;
str << " GType# " << GType.ToString();
str << "}";
return str.Str();
@@ -295,7 +295,7 @@ namespace NKikimr {
WriteSpeedBps = std::min(WriteSpeedBps, other.WriteSpeedBps);
ReadBlockSize = std::min(ReadBlockSize, other.ReadBlockSize);
WriteBlockSize = std::min(WriteBlockSize, other.WriteBlockSize);
- MinHugeBlobInBytes = std::max(MinHugeBlobInBytes, other.MinHugeBlobInBytes);
+ MinREALHugeBlobInBytes = std::max(MinREALHugeBlobInBytes, other.MinREALHugeBlobInBytes);
}
// PDisk messages cost
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h
index 51887b8ae9..16ae661627 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_costmodel.h
@@ -58,11 +58,11 @@ namespace NKikimr {
ui64 WriteSpeedBps;
ui64 ReadBlockSize;
ui64 WriteBlockSize;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
TBlobStorageGroupType GType;
TCostModel(ui64 seekTimeUs, ui64 readSpeedBps, ui64 writeSpeedBps, ui64 readBlockSize, ui64 writeBlockSize,
- ui32 minHugeBlobInBytes, TBlobStorageGroupType gType);
+ ui32 minREALHugeBlobInBytes, TBlobStorageGroupType gType);
TCostModel(const NKikimrBlobStorage::TVDiskCostSettings &settings, TBlobStorageGroupType gType);
/// SETTINGS
@@ -95,7 +95,7 @@ namespace NKikimr {
WriteSpeedBps != other.WriteSpeedBps ||
ReadBlockSize != other.ReadBlockSize ||
WriteBlockSize != other.WriteBlockSize ||
- MinHugeBlobInBytes != other.MinHugeBlobInBytes;
+ MinREALHugeBlobInBytes != other.MinREALHugeBlobInBytes;
}
// PDisk messages cost
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_events.h b/ydb/core/blobstorage/vdisk/common/vdisk_events.h
index 05d89c5080..b99db787e0 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_events.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_events.h
@@ -703,8 +703,8 @@ namespace NKikimr {
if (costSettings.HasWriteBlockSize()) {
str << " WriteBlockSize# " << costSettings.GetWriteBlockSize();
}
- if (costSettings.HasMinHugeBlobInBytes()) {
- str << " MinHugeBlobInBytes# " << costSettings.GetMinHugeBlobInBytes();
+ if (costSettings.HasMinREALHugeBlobInBytes()) {
+ str << " MinREALHugeBlobInBytes# " << costSettings.GetMinREALHugeBlobInBytes();
}
str << "}";
}
@@ -3199,9 +3199,9 @@ namespace NKikimr {
////////////////////////////////////////////////////////////////////////////
class TEvMinHugeBlobSizeUpdate : public TEventLocal<TEvMinHugeBlobSizeUpdate, TEvBlobStorage::EvMinHugeBlobSizeUpdate> {
public:
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
- TEvMinHugeBlobSizeUpdate(ui32 minHugeBlobInBytes) : MinHugeBlobInBytes(minHugeBlobInBytes) {
+ TEvMinHugeBlobSizeUpdate(ui32 minREALHugeBlobInBytes) : MinREALHugeBlobInBytes(minREALHugeBlobInBytes) {
};
};
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp
index d74fcafb7d..25c64e1d73 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.cpp
@@ -8,18 +8,18 @@ namespace NKikimr {
///////////////////////////////////////////////////////////////////////////////////
namespace NPriPut {
- EHandleType HandleType(const ui32 minHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
+ EHandleType HandleType(const ui32 minREALHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
ui32 originalBufSizeWithoutOverhead, bool addHeader) {
// what size of huge blob it would be, if it huge
const ui64 hugeBlobSize = (addHeader ? TDiskBlob::HeaderSize : 0) + originalBufSizeWithoutOverhead;
switch (handleClass) {
case NKikimrBlobStorage::TabletLog:
- return (hugeBlobSize >= minHugeBlobSize ? HugeForeground : Log);
+ return (hugeBlobSize >= minREALHugeBlobSize ? HugeForeground : Log);
case NKikimrBlobStorage::AsyncBlob:
- return (hugeBlobSize >= minHugeBlobSize ? HugeBackground : Log);
+ return (hugeBlobSize >= minREALHugeBlobSize ? HugeBackground : Log);
case NKikimrBlobStorage::UserData:
- return (hugeBlobSize >= minHugeBlobSize ? HugeForeground : Log);
+ return (hugeBlobSize >= minREALHugeBlobSize ? HugeForeground : Log);
default:
Y_ABORT("Unexpected case");
}
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h
index 73706918ed..eb86e0b36f 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_handle_class.h
@@ -17,7 +17,7 @@ namespace NKikimr {
HugeBackground = 2 // huge blog, write it with low priority
};
- EHandleType HandleType(const ui32 minHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
+ EHandleType HandleType(const ui32 minREALHugeBlobSize, NKikimrBlobStorage::EPutHandleClass handleClass,
ui32 originalBufSizeWithoutOverhead, bool addHeader);
} // NPriPut
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
index 2c581f9fbf..f8d4b2fec4 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.cpp
@@ -3,23 +3,23 @@
namespace NKikimr {
- THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo,
- TSearchTable &&searchTable)
+ THugeSlotsMap::THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable)
: AppendBlockSize(appendBlockSize)
- , MinHugeBlobInBlocks(minHugeBlobInBlocks)
, AllSlotsInfo(std::move(slotsInfo))
, SearchTable(std::move(searchTable))
{}
const THugeSlotsMap::TSlotInfo *THugeSlotsMap::GetSlotInfo(ui32 size) const {
- const ui32 sizeInBlocks = (size + AppendBlockSize - 1) / AppendBlockSize;
- Y_ABORT_UNLESS(MinHugeBlobInBlocks <= sizeInBlocks);
- const ui64 idx = SearchTable.at(sizeInBlocks - MinHugeBlobInBlocks);
+ ui32 sizeInBlocks = size / AppendBlockSize;
+ sizeInBlocks += !(sizeInBlocks * AppendBlockSize == size);
+ const ui64 idx = SearchTable.at(sizeInBlocks);
return &AllSlotsInfo.at(idx);
}
ui32 THugeSlotsMap::AlignByBlockSize(ui32 size) const {
- return Max(MinHugeBlobInBlocks * AppendBlockSize, size - size % AppendBlockSize);
+ ui32 sizeInBlocks = size / AppendBlockSize;
+ Y_ABORT_UNLESS(sizeInBlocks, "Blob size to align is smaller than a single block. BlobSize# %" PRIu32, size);
+ return sizeInBlocks * AppendBlockSize;
}
void THugeSlotsMap::Output(IOutputStream &str) const {
@@ -31,7 +31,11 @@ namespace NKikimr {
str << "]}\n";
str << "{SearchTable# [";
for (const auto &idx : SearchTable) {
- AllSlotsInfo.at(idx).Output(str);
+ if (idx != NoOpIdx) {
+ AllSlotsInfo.at(idx).Output(str);
+ } else {
+ str << "null";
+ }
str << "\n";
}
str << "]}";
@@ -44,9 +48,9 @@ namespace NKikimr {
}
// check whether this blob is huge one; userPartSize doesn't include any metadata stored along with blob
- bool THugeBlobCtx::IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minHugeBlobInBytes) const {
- return gtype.MaxPartSize(fullId) + (AddHeader ? TDiskBlob::HeaderSize : 0) >= minHugeBlobInBytes;
- }
+ bool THugeBlobCtx::IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minREALHugeBlobInBytes) const {
+ return gtype.MaxPartSize(fullId) + (AddHeader ? TDiskBlob::HeaderSize : 0) >= minREALHugeBlobInBytes;
+ }
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
index 1d7e61f04e..5a26e1e16f 100644
--- a/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
+++ b/ydb/core/blobstorage/vdisk/common/vdisk_hugeblobctx.h
@@ -38,13 +38,16 @@ namespace NKikimr {
};
// All slot types
- using TAllSlotsInfo = std::vector<TSlotInfo>;
+ using TAllSlotsInfo = TVector<TSlotInfo>;
// Type to address TAllSlotsInfo
using TIndex = ui16;
// Size in AppendBlockSize -> index in TAllSlotsInfo
- using TSearchTable = std::vector<TIndex>;
+ using TSearchTable = TVector<TIndex>;
+ // Idx that indicates there is no record for it in TAllSlotsInfo
+ static constexpr TIndex NoOpIdx = Max<TIndex>();
- THugeSlotsMap(ui32 appendBlockSize, ui32 minHugeBlobInBlocks, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
+
+ THugeSlotsMap(ui32 appendBlockSize, TAllSlotsInfo &&slotsInfo, TSearchTable &&searchTable);
const TSlotInfo *GetSlotInfo(ui32 size) const;
ui32 AlignByBlockSize(ui32 size) const;
void Output(IOutputStream &str) const;
@@ -52,7 +55,6 @@ namespace NKikimr {
private:
const ui32 AppendBlockSize;
- const ui32 MinHugeBlobInBlocks;
TAllSlotsInfo AllSlotsInfo;
TSearchTable SearchTable;
};
@@ -66,7 +68,7 @@ namespace NKikimr {
const bool AddHeader;
// check whether this NEW blob is huge one; userPartSize doesn't include any metadata stored along with blob
- bool IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minHugeBlobInBytes) const;
+ bool IsHugeBlob(TBlobStorageGroupType gtype, const TLogoBlobID& fullId, ui32 minREALHugeBlobInBytes) const;
THugeBlobCtx(const std::shared_ptr<const THugeSlotsMap> &hugeSlotsMap, bool addHeader)
: HugeSlotsMap(hugeSlotsMap)
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
index 699f5ca6ef..2aa98ea92c 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhuge_ut.cpp
@@ -17,6 +17,7 @@ namespace NKikimr {
Y_UNIT_TEST(SerializeParse) {
ui32 chunkSize = 134274560u;
ui32 appendBlockSize = 56896u;
+ ui32 minHugeBlobInBytes = 512u << 10u;
ui32 milestoneHugeBlobInBytes = 512u << 10u;
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
@@ -28,8 +29,8 @@ namespace NKikimr {
auto vctx = MakeIntrusive<TVDiskContext>(TActorId(), info->PickTopology(), counters, TVDiskID(0, 1, 0, 0, 0),
nullptr, NPDisk::DEVICE_TYPE_UNKNOWN);
std::unique_ptr<THullHugeKeeperPersState> state(
- new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize,
- appendBlockSize, milestoneHugeBlobInBytes, maxBlobInBytes,
+ new THullHugeKeeperPersState(vctx, chunkSize, appendBlockSize, appendBlockSize,
+ minHugeBlobInBytes, milestoneHugeBlobInBytes, maxBlobInBytes,
overhead, freeChunksReservation, logf));
state->LogPos = THullHugeRecoveryLogPos(0, 0, 100500, 50000, 70000, 56789, 39482);
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp
index a6ff46e107..be27874e3f 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp
@@ -2,8 +2,6 @@
#include <library/cpp/monlib/service/pages/templates.h>
-#include <ranges>
-
namespace NKikimr {
namespace NHuge {
@@ -26,7 +24,7 @@ namespace NKikimr {
}
void TChainLayoutBuilder::Output(IOutputStream &str, ui32 appendBlockSize) const {
- str << "CHAIN TABLE (MilestoneId=" << MilestoneId << " rows=" << Layout.size() << "):\n";
+ str << "CHAIN TABLE (MilesoneId=" << MilesoneId << " rows=" << Layout.size() << "):\n";
for (const auto &x : Layout) {
str << "Blocks# (" << x.Left << ", " << x.Right << "]";
if (appendBlockSize) {
@@ -66,7 +64,7 @@ namespace NKikimr {
}
void TChainLayoutBuilder::BuildUpward(ui32 left, ui32 right, ui32 overhead) {
- MilestoneId = Layout.size();
+ MilesoneId = Layout.size();
ui32 valBlocks = left;
ui32 shiftBlocks = 0;
@@ -85,21 +83,6 @@ namespace NKikimr {
////////////////////////////////////////////////////////////////////////////
// TChain
////////////////////////////////////////////////////////////////////////////
- THugeSlot TChain::Convert(const NPrivate::TChunkSlot& id) const {
- return THugeSlot(id.GetChunkId(), id.GetSlotId() * SlotSize, SlotSize);
- }
-
- NPrivate::TChunkSlot TChain::Convert(const TDiskPart& addr) const {
- ui32 slotId = addr.Offset / SlotSize;
- Y_VERIFY_S(slotId * SlotSize == addr.Offset, VDiskLogPrefix << "slotId# " << slotId
- << " addr# " << addr.ToString() << " State# " << ToString());
- return NPrivate::TChunkSlot(addr.ChunkIdx, slotId);
- }
-
- NPrivate::TChunkSlot TChain::Convert(const THugeSlot& slot) const {
- return Convert(slot.GetDiskPart());
- }
-
TMask TChain::BuildConstMask(const TString &prefix, ui32 slotsInChunk) {
Y_VERIFY_S(1 < slotsInChunk && slotsInChunk <= MaxNumberOfSlots,
prefix << "It's not a good idea to have so many slots in chunk;"
@@ -267,114 +250,73 @@ namespace NKikimr {
void TChain::Save(IOutputStream *s) const {
::Save(s, SlotsInChunk);
::Save(s, AllocatedSlots);
- ::SaveSize(s, FreeSpace.size() + LockedChunks.size());
- ForEachFreeSpaceChunk(std::bind(&::Save<TFreeSpace::value_type>, s, std::placeholders::_1));
+ if (LockedChunks) {
+ TFreeSpace temp(FreeSpace);
+ temp.insert(LockedChunks.begin(), LockedChunks.end());
+ ::Save(s, temp);
+ } else {
+ ::Save(s, FreeSpace);
+ }
}
- TChain TChain::Load(IInputStream *s, TString vdiskLogPrefix, ui32 appendBlockSize, ui32 blocksInChunk,
- std::span<TChain> chains, bool *compatible) {
- ui32 slotsInChunk;
+ void TChain::Load(IInputStream *s) {
+ FreeSpace.clear();
+ ui32 slotsInChunk = 0;
::Load(s, slotsInChunk);
-
- const ui32 slotSizeInBlocks = blocksInChunk / slotsInChunk;
- Y_ABORT_UNLESS(slotSizeInBlocks);
- ui32 slotSize = slotSizeInBlocks * appendBlockSize; // assume optimal slot size for specific slots per chunk
-
- // check if this goes with compatible chain
- for (const TChain& chainFromBuilder : chains) {
- if (chainFromBuilder.SlotsInChunk < slotsInChunk) { // no such chain from builder
- *compatible = false;
- } else if (chainFromBuilder.SlotsInChunk == slotsInChunk) {
- slotSize = chainFromBuilder.SlotSize;
- } else {
- continue;
- }
- break;
- }
-
- TChain res{
- std::move(vdiskLogPrefix),
- slotsInChunk,
- slotSize, // in bytes
- };
-
- ::Load(s, res.AllocatedSlots);
- ::Load(s, res.FreeSpace);
- for (const auto& [chunkId, mask] : res.FreeSpace) {
- res.FreeSlotsInFreeSpace += mask.Count();
+ Y_VERIFY_S(slotsInChunk == SlotsInChunk, VDiskLogPrefix
+ << "slotsInChunk# " << slotsInChunk << " SlotsInChunk# " << SlotsInChunk);
+ ::Load(s, AllocatedSlots);
+ ::Load(s, FreeSpace);
+ FreeSlotsInFreeSpace = 0;
+ for (const auto &[chunkId, mask] : FreeSpace) {
+ // all 1 in mask -- free slots
+ // 0 - slot is in use
+ FreeSlotsInFreeSpace += mask.Count();
}
-
- return res;
}
bool TChain::HaveBeenUsed() const {
- return AllocatedSlots; // chain is considered to be used if it contains any allocated slots
+ return AllocatedSlots != 0 || !FreeSpace.empty();
}
TString TChain::ToString() const {
TStringStream str;
-
- str << "{" << SlotSize << '/' << SlotsInChunk << " AllocatedSlots# " << AllocatedSlots << " Free#";
-
- bool any = false;
- ForEachFreeSpaceChunk([&](const auto& value) {
- const auto& [chunk, bitmap] = value;
- str << " {" << chunk;
- ui32 begin;
- ui32 prev = Max<ui32>();
- Y_FOR_EACH_BIT(i, bitmap) {
- if (prev == Max<ui32>()) {
- begin = i;
- } else if (i != prev + 1) {
- str << ' ' << begin;
- if (begin != prev) {
- str << '-' << prev;
- }
- begin = prev;
+ auto output = [&str] (const TFreeSpace &c) {
+ for (const auto &x : c) {
+ for (size_t i = 0; i < x.second.Size(); i++) {
+ if (x.second.Test(i))
+ str << " [" << x.first << " " << i << "]";
}
- prev = i;
- }
- str << ' ' << begin;
- if (begin != prev) {
- str << '-' << prev;
}
- str << '}';
-
- any = true;
- });
-
- if (!any) {
- str << " none";
- }
+ };
+ str << "{AllocatedSlots# " << AllocatedSlots << " [ChunkId FreeSlot]:";
+ output(FreeSpace);
+ output(LockedChunks);
str << "}";
return str.Str();
}
void TChain::RenderHtml(IOutputStream &str) const {
- HTML(str) {
- TABLER() {
- TABLED() {
- str << SlotSize << "/" << SlotsInChunk;
+ auto output = [&str] (const TFreeSpace &c) {
+ for (const auto &x : c) {
+ size_t freeSlots = 0;
+ for (size_t i = 0; i < x.second.Size(); i++) {
+ if (x.second.Test(i))
+ ++freeSlots;
}
- TABLED() {
- ForEachFreeSpaceChunk([&](const auto& value) {
- const auto& [chunk, bitmap] = value;
- str << " [" << chunk << " " << bitmap.Count() << "]";
- });
+ if (freeSlots) {
+ str << " [" << x.first << " " << freeSlots << "]";
}
}
- }
+ };
+
+ output(FreeSpace);
+ output(LockedChunks);
}
- void TChain::RenderHtmlForUsage(IOutputStream &str) const {
- HTML(str) {
- TABLER() {
- TABLED() { str << SlotSize; }
- TABLED() { str << SlotsInChunk; }
- TABLED() { str << AllocatedSlots; }
- }
- }
+ ui32 TChain::GetAllocatedSlots() const {
+ return AllocatedSlots;
}
void TChain::GetOwnedChunks(TSet<TChunkIdx>& chunks) const {
@@ -388,6 +330,87 @@ namespace NKikimr {
}
////////////////////////////////////////////////////////////////////////////
+ // TChainDelegator
+ ////////////////////////////////////////////////////////////////////////////
+ TChainDelegator::TChainDelegator(const TString &vdiskLogPrefix, ui32 valBlocks, ui32 shiftBlocks,
+ ui32 chunkSize, ui32 appendBlockSize)
+ : VDiskLogPrefix(vdiskLogPrefix)
+ , Blocks(valBlocks)
+ , ShiftInBlocks(shiftBlocks)
+ , SlotsInChunk(0)
+ , SlotSize(0)
+ {
+ ui32 slotSizeInBlocks = Blocks + ShiftInBlocks;
+ ui32 blocksInChunk = chunkSize / appendBlockSize;
+ Y_VERIFY_S(appendBlockSize * blocksInChunk == chunkSize, VDiskLogPrefix
+ << "Blocks# " << Blocks << " ShiftInBlocks# " << ShiftInBlocks
+ << " chunkSize# " << chunkSize << " appendBlockSize# " << appendBlockSize);
+
+ SlotsInChunk = blocksInChunk / slotSizeInBlocks;
+ SlotSize = slotSizeInBlocks * appendBlockSize;
+
+ ChainPtr = MakeIntrusive<TChain>(vdiskLogPrefix, SlotsInChunk);
+ }
+
+ THugeSlot TChainDelegator::Convert(const NPrivate::TChunkSlot &id) const {
+ return THugeSlot(id.GetChunkId(), id.GetSlotId() * SlotSize, SlotSize);
+ }
+
+ NPrivate::TChunkSlot TChainDelegator::Convert(const TDiskPart &addr) const {
+ ui32 slotId = addr.Offset / SlotSize;
+ Y_VERIFY_S(slotId * SlotSize == addr.Offset, VDiskLogPrefix
+ << "slotId# " << slotId << " addr# " << addr.ToString() << " State# " << ToString());
+ return NPrivate::TChunkSlot(addr.ChunkIdx, slotId);
+ }
+
+ NPrivate::TChunkSlot TChainDelegator::Convert(const THugeSlot &slot) const {
+ return Convert(slot.GetDiskPart());
+ }
+
+ void TChainDelegator::Save(IOutputStream *s) const {
+ ::Save(s, *ChainPtr);
+ }
+
+ void TChainDelegator::Load(IInputStream *s) {
+ ::Load(s, *ChainPtr);
+ }
+
+ bool TChainDelegator::HaveBeenUsed() const {
+ return ChainPtr->HaveBeenUsed();
+ }
+
+ TString TChainDelegator::ToString() const {
+ TStringStream str;
+ str << "{[SlotSize, SlotsInChunk]: [" << SlotSize << ", " << SlotsInChunk << "] "
+ << ChainPtr->ToString() << "}";
+ return str.Str();
+ }
+
+ void TChainDelegator::GetOwnedChunks(TSet<TChunkIdx>& chunks) const {
+ ChainPtr->GetOwnedChunks(chunks);
+ }
+
+ void TChainDelegator::RenderHtml(IOutputStream &str) const {
+ HTML(str) {
+ TABLER() {
+ TABLED() {str << SlotSize << " / " << SlotsInChunk;}
+ TABLED() {ChainPtr->RenderHtml(str);}
+ }
+ }
+ }
+
+ void TChainDelegator::RenderHtmlForUsage(IOutputStream &str) const {
+ HTML(str) {
+ TABLER() {
+ TABLED() {str << SlotSize;}
+ TABLED() {str << SlotsInChunk;}
+ TABLED() {str << ChainPtr->GetAllocatedSlots();}
+ }
+ }
+ }
+
+
+ ////////////////////////////////////////////////////////////////////////////
// TAllChains
////////////////////////////////////////////////////////////////////////////
TAllChains::TAllChains(
@@ -395,6 +418,7 @@ namespace NKikimr {
ui32 chunkSize,
ui32 appendBlockSize,
ui32 minHugeBlobInBytes,
+ ui32 oldMinHugeBlobSizeInBytes,
ui32 milestoneBlobInBytes,
ui32 maxBlobInBytes,
ui32 overhead)
@@ -402,137 +426,143 @@ namespace NKikimr {
, ChunkSize(chunkSize)
, AppendBlockSize(appendBlockSize)
, MinHugeBlobInBytes(minHugeBlobInBytes)
+ , OldMinHugeBlobSizeInBytes(oldMinHugeBlobSizeInBytes)
, MilestoneBlobInBytes(milestoneBlobInBytes)
, MaxBlobInBytes(maxBlobInBytes)
, Overhead(overhead)
- , MinHugeBlobInBlocks(MinHugeBlobInBytes / AppendBlockSize)
{
- Y_VERIFY_S(MinHugeBlobInBytes &&
+ Y_VERIFY_S(MinHugeBlobInBytes != 0 &&
+ MinHugeBlobInBytes >= AppendBlockSize &&
MinHugeBlobInBytes <= MilestoneBlobInBytes &&
+ MinHugeBlobInBytes <= OldMinHugeBlobSizeInBytes &&
MilestoneBlobInBytes < MaxBlobInBytes, "INVALID CONFIGURATION! (SETTINGS ARE:"
- << " MaxBlobInBytes# " << MaxBlobInBytes << " MinHugeBlobInBytes# " << MinHugeBlobInBytes
+ << " MaxBlobInBytes# " << MaxBlobInBytes << " MinHugeBlobInBytes# " << MinHugeBlobInBytes << " OldMinHugeBlobSizeInBytes# " << OldMinHugeBlobSizeInBytes
<< " MilestoneBlobInBytes# " << MilestoneBlobInBytes << " ChunkSize# " << ChunkSize
<< " AppendBlockSize# " << AppendBlockSize << ")");
-
- BuildChains();
+ BuildLayout();
}
- TChain *TAllChains::GetChain(ui32 size) {
- if (size < MinHugeBlobInBytes || GetEndBlocks() * AppendBlockSize < size) {
- return nullptr;
- }
- const size_t index = SizeToBlocks(size) - MinHugeBlobInBlocks;
- Y_DEBUG_ABORT_UNLESS(index < SearchTable.size());
- const size_t chainIndex = SearchTable[index];
- Y_DEBUG_ABORT_UNLESS(chainIndex < Chains.size());
- return &Chains[chainIndex];
+ TChainDelegator *TAllChains::GetChain(ui32 size) {
+ return SearchTable.at(SizeToBlocks(size));
}
- const TChain *TAllChains::GetChain(ui32 size) const {
- if (size < MinHugeBlobInBytes || GetEndBlocks() * AppendBlockSize < size) {
- return nullptr;
- }
- Y_ABORT_UNLESS(MinHugeBlobInBytes <= size);
- const size_t index = SizeToBlocks(size) - MinHugeBlobInBlocks;
- Y_DEBUG_ABORT_UNLESS(index < SearchTable.size());
- const size_t chainIndex = SearchTable[index];
- Y_DEBUG_ABORT_UNLESS(chainIndex < Chains.size());
- return &Chains[chainIndex];
+ const TChainDelegator *TAllChains::GetChain(ui32 size) const {
+ return SearchTable.at(SizeToBlocks(size));
}
THeapStat TAllChains::GetStat() const {
THeapStat stat;
- for (const auto& chain : Chains) {
- stat += chain.GetStat();
+ for (const auto &x : ChainDelegators) {
+ stat += x.ChainPtr->GetStat();
}
return stat;
}
- void TAllChains::Save(IOutputStream *s) const {
- // check if we can write compatible entrypoint (with exactly the same set of chains)
- bool writeCompatible = true;
- ui32 numChains = 0;
- if (DeserializedSlotSizes.empty()) { // if this was initially empty heap, write it fully
- writeCompatible = false;
- } else {
- for (auto& chain : Chains) {
- if (DeserializedSlotSizes.contains(chain.SlotSize)) {
- ++numChains;
- } else if (chain.HaveBeenUsed()) {
- writeCompatible = false;
- break;
- }
- }
+ void TAllChains::PrintOutChains(IOutputStream &str) const {
+ str << "CHAIN TABLE (rows=" << ChainDelegators.size() << "):\n";
+ for (const auto &x : ChainDelegators) {
+ str << "Blocks# (" << x.Blocks << ", " << (x.Blocks + x.ShiftInBlocks) << "] "
+ << "Bytes# (" << (x.Blocks * AppendBlockSize) << ", "
+ << ((x.Blocks + x.ShiftInBlocks) * AppendBlockSize) << "] "
+ << " SlotSize# " << x.SlotSize << " SlotsInChunk# " << x.SlotsInChunk << "\n";
}
+ }
- if (!writeCompatible) { // we can't, so we serialize all our chains anyway
- numChains = Chains.size();
+ void TAllChains::PrintOutSearchTable(IOutputStream &str) const {
+ str << "SEARCH TABLE:\n";
+ for (const auto &x : SearchTable) {
+ str << (&x - &SearchTable[0]) << " idx: ";
+ if (x) {
+ str << x - &ChainDelegators[0];
+ } else {
+ str << "null";
+ }
+ str << "\n";
}
- ::Save(s, numChains);
-
- // serialize selected chains
- auto chains = Chains | std::views::filter([&](const auto& chain) {
- return !writeCompatible || DeserializedSlotSizes.contains(chain.SlotSize);
- });
- ::SaveRange(s, chains.begin(), chains.end());
}
- void TAllChains::Load(IInputStream *s) {
- std::vector<TChain> newChains;
- newChains.reserve(Chains.size());
-
- auto chainsIt = Chains.begin();
- const auto chainsEnd = Chains.end();
-
- Y_DEBUG_ABORT_UNLESS(ChunkSize % AppendBlockSize == 0);
- Y_DEBUG_ABORT_UNLESS(AppendBlockSize <= ChunkSize);
- const ui32 blocksInChunk = ChunkSize / AppendBlockSize;
-
- bool compatible = true; // if the entrypoint is compatible with chain layout builder
- ui32 prevSlotSize = 0;
-
- ui32 size;
- for (::Load(s, size); size; --size) {
- auto chain = TChain::Load(s, VDiskLogPrefix, AppendBlockSize, blocksInChunk, {chainsIt, chainsEnd},
- &compatible);
- DeserializedSlotSizes.insert(chain.SlotSize);
-
- Y_ABORT_UNLESS(chain.SlotSize > std::exchange(prevSlotSize, chain.SlotSize));
-
- for (; chainsIt != chainsEnd && chainsIt->SlotSize <= chain.SlotSize; ++chainsIt) {
- if (chainsIt->SlotSize == chain.SlotSize) {
- Y_ABORT_UNLESS(chain.SlotsInChunk == chainsIt->SlotsInChunk);
- Y_ABORT_UNLESS(!chainsIt->HaveBeenUsed());
- } else {
- newChains.push_back(std::move(*chainsIt));
+ void TAllChains::Save(IOutputStream *s) const {
+ if (StartMode == EStartMode::Loaded) {
+ ui32 size = ChainDelegators.size();
+ ::Save(s, size);
+ for (auto& d : ChainDelegators) {
+ ::Save(s, d);
+ }
+ } else {
+ std::vector<const TChainDelegator*> delegators;
+ for (auto &x : ChainDelegators) {
+ if (!x.HaveBeenUsed() && x.SlotSize < FirstLoadedSlotSize) {
+ continue; // preserving backward compatibility until no allocations
}
+ delegators.emplace_back(&x);
}
- newChains.push_back(std::move(chain));
+ ui32 size = delegators.size();
+ ::Save(s, size);
+ for (auto x : delegators) {
+ ::Save(s, *x);
+ }
}
+ }
- std::ranges::move(chainsIt, chainsEnd, std::back_inserter(newChains));
+ void TAllChains::Load(IInputStream *s) {
+ ui32 size = 0;
+ // load array size
+ ::Load(s, size);
+ if (size == ChainDelegators.size()) {
+ StartMode = EStartMode::Loaded;
+ // load map and current map are of the same size, just load it
+ for (auto &x : ChainDelegators) {
+ ::Load(s, x);
+ }
+ } else if (size < ChainDelegators.size()) {
+ // map size has been changed, run migration
+ StartMode = EStartMode::Migrated;
+ TAllChainDelegators chainDelegators = BuildChains(OldMinHugeBlobSizeInBytes);
+ Y_VERIFY_S(size > 0 && size == chainDelegators.size(), "size# " << size
+ << " chainDelegators.size()# " << chainDelegators.size());
+
+ // load into temporary delegators
+ for (auto &x : chainDelegators) {
+ ::Load(s, x);
+ }
- Chains = std::move(newChains);
- if (!compatible) { // deserialized slot sizes can't be stored in compatible way
- DeserializedSlotSizes.clear();
+ // migrate
+ using TIt = TAllChainDelegators::iterator;
+ TIt loadedIt = chainDelegators.begin();
+ TIt loadedEnd = chainDelegators.end();
+ FirstLoadedSlotSize = loadedIt->SlotSize;
+ for (TIt it = ChainDelegators.begin(); it != ChainDelegators.end(); ++it) {
+ Y_ABORT_UNLESS(loadedIt != loadedEnd);
+ if (loadedIt->SlotSize == it->SlotSize) {
+ *it = std::move(*loadedIt);
+ ++loadedIt;
+ }
+ }
+ Y_ABORT_UNLESS(loadedIt == loadedEnd);
+ } else {
+ // entry point size rollback case
+ Y_ABORT_UNLESS(size > ChainDelegators.size());
+ ui32 curChainDelegatorsSize = ChainDelegators.size();
+ Y_FAIL_S("Impossible case; MinHugeBlobInBytes# " << MinHugeBlobInBytes
+ << " MilestoneBlobInBytes# " << MilestoneBlobInBytes
+ << " loadedSize# " << size
+ << " curChainDelegatorsSize# " << curChainDelegatorsSize);
}
}
void TAllChains::GetOwnedChunks(TSet<TChunkIdx>& chunks) const {
- for (const TChain& chain : Chains) {
- chain.GetOwnedChunks(chunks);
+ for (const TChainDelegator& delegator : ChainDelegators) {
+ delegator.GetOwnedChunks(chunks);
}
}
TString TAllChains::ToString() const {
TStringStream str;
- str << "{ChunkSize# " << ChunkSize
- << " AppendBlockSize# " << AppendBlockSize
- << " MinHugeBlobInBytes# " << MinHugeBlobInBytes
- << " MaxBlobInBytes# " << MaxBlobInBytes;
- for (const auto& chain : Chains) {
- str << " {CHAIN " << chain.ToString() << "}";
+ str << "{ChunkSize# " << ChunkSize << " AppendBlockSize# " << AppendBlockSize
+ << " MinHugeBlobInBytes# " << MinHugeBlobInBytes << " MaxBlobInBytes# " << MaxBlobInBytes;
+ for (const auto & x : ChainDelegators) {
+ str << " {CHAIN " << x.ToString() << "}";
}
str << "}";
return str.Str();
@@ -548,9 +578,8 @@ namespace NKikimr {
}
}
TABLEBODY() {
- for (const auto& chain : Chains) {
- chain.RenderHtml(str);
- }
+ for (const auto & x : ChainDelegators)
+ x.RenderHtml(str);
}
}
}
@@ -567,9 +596,8 @@ namespace NKikimr {
}
}
TABLEBODY() {
- for (const auto& chain : Chains) {
- chain.RenderHtmlForUsage(str);
- }
+ for (const auto & x : ChainDelegators)
+ x.RenderHtmlForUsage(str);
}
}
}
@@ -577,87 +605,91 @@ namespace NKikimr {
TVector<NPrivate::TChainLayoutBuilder::TSeg> TAllChains::GetLayout() const {
TVector<NPrivate::TChainLayoutBuilder::TSeg> res;
- res.reserve(Chains.size());
- ui32 prevSlotSizeInBlocks = MinHugeBlobInBlocks;
- for (const auto& chain : Chains) {
- const ui32 slotSizeInBlocks = chain.SlotSize / AppendBlockSize;
- res.push_back({
- prevSlotSizeInBlocks,
- slotSizeInBlocks - prevSlotSizeInBlocks,
- });
- prevSlotSizeInBlocks = slotSizeInBlocks;
+ res.reserve(ChainDelegators.size());
+ for (const auto &x : ChainDelegators) {
+ res.push_back(NPrivate::TChainLayoutBuilder::TSeg {x.Blocks, x.Blocks + x.ShiftInBlocks} );
}
return res;
}
std::shared_ptr<THugeSlotsMap> TAllChains::BuildHugeSlotsMap() const {
- THugeSlotsMap::TAllSlotsInfo allSlotsInfo;
- for (const auto& chain : Chains) {
- allSlotsInfo.emplace_back(chain.SlotSize, chain.SlotsInChunk);
+ THugeSlotsMap::TAllSlotsInfo targetAllSlotsInfo;
+ THugeSlotsMap::TSearchTable targetSearchTable;
+
+ for (const auto &x : SearchTable) {
+ if (!x) {
+ // first records in SearchTable are equal to nullptr
+ targetSearchTable.push_back(THugeSlotsMap::NoOpIdx);
+ continue;
+ }
+
+ if (targetAllSlotsInfo.empty() || targetAllSlotsInfo.back().SlotSize != x->SlotSize) {
+ targetAllSlotsInfo.emplace_back(x->SlotSize, x->SlotsInChunk);
+ }
+ targetSearchTable.push_back(THugeSlotsMap::TIndex(targetAllSlotsInfo.size() - 1));
}
- return std::make_shared<THugeSlotsMap>(AppendBlockSize, MinHugeBlobInBlocks, std::move(allSlotsInfo),
- THugeSlotsMap::TSearchTable(SearchTable));
+
+ return std::make_shared<THugeSlotsMap>(AppendBlockSize, std::move(targetAllSlotsInfo),
+ std::move(targetSearchTable));
}
////////////////////////////////////////////////////////////////////////////
// TAllChains: Private
////////////////////////////////////////////////////////////////////////////
- void TAllChains::BuildChains() {
- const ui32 startBlocks = MinHugeBlobInBlocks;
- const ui32 milestoneBlocks = MilestoneBlobInBytes / AppendBlockSize;
+ TAllChains::TAllChainDelegators TAllChains::BuildChains(ui32 minHugeBlobInBytes) const {
+ // minHugeBlobInBytes -- is the only variable parameter, used for migration
+ const ui32 startBlocks = minHugeBlobInBytes / AppendBlockSize;
+ const ui32 mileStoneBlocks = MilestoneBlobInBytes / AppendBlockSize;
const ui32 endBlocks = GetEndBlocks();
- NPrivate::TChainLayoutBuilder builder(startBlocks, milestoneBlocks, endBlocks, Overhead);
- const ui32 blocksInChunk = ChunkSize / AppendBlockSize;
+ NPrivate::TChainLayoutBuilder builder(startBlocks, mileStoneBlocks, endBlocks, Overhead);
+ Y_ABORT_UNLESS(!builder.GetLayout().empty());
+ TAllChainDelegators result;
for (auto x : builder.GetLayout()) {
- const ui32 slotSizeInBlocks = x.Right;
- const ui32 slotSize = slotSizeInBlocks * AppendBlockSize;
- const ui32 slotsInChunk = blocksInChunk / slotSizeInBlocks;
- Chains.emplace_back(VDiskLogPrefix, slotsInChunk, slotSize);
+ result.emplace_back(VDiskLogPrefix, x.Left, x.Right - x.Left,
+ ChunkSize, AppendBlockSize);
}
-
- Y_ABORT_UNLESS(!Chains.empty());
+ return result;
}
void TAllChains::BuildSearchTable() {
- Y_ABORT_UNLESS(SearchTable.empty());
- Y_ABORT_UNLESS(!Chains.empty());
-
- const ui32 startBlocks = MinHugeBlobInBlocks;
- const ui32 minSize = startBlocks * AppendBlockSize;
- const ui32 endBlocks = GetEndBlocks(); // maximum possible number of blocks per huge blob
- auto it = Chains.begin();
- ui16 index = 0;
-
- SearchTable.reserve(endBlocks - startBlocks + 1);
- for (ui32 i = startBlocks, size = minSize; i <= endBlocks; ++i, size += AppendBlockSize) {
- if (it->SlotSize < size) { // size doesn't fit in current chain, but it must fit into next one
+ const ui32 endBlocks = GetEndBlocks();
+ Y_DEBUG_ABORT_UNLESS(!ChainDelegators.empty());
+ TAllChainDelegators::iterator it = ChainDelegators.begin();
+ TChainDelegator *ptr = nullptr;
+ ui32 blocks = it->Blocks;
+ for (ui32 i = 0; i <= endBlocks; i++) {
+ if (i <= blocks) {
+ } else {
+ ptr = &(*it);
++it;
- Y_ABORT_UNLESS(it != Chains.end());
- Y_ABORT_UNLESS(size <= it->SlotSize);
- Y_ABORT_UNLESS(index != Max<ui16>());
- ++index;
+ if (it == ChainDelegators.end())
+ blocks = ui32(-1);
+ else
+ blocks = it->Blocks;
}
- SearchTable.push_back(index);
+ SearchTable.push_back(ptr);
}
}
- ui32 TAllChains::SizeToBlocks(ui32 size) const {
- return (size + AppendBlockSize - 1) / AppendBlockSize;
+ void TAllChains::BuildLayout()
+ {
+ ChainDelegators = BuildChains(MinHugeBlobInBytes);
+ Y_ABORT_UNLESS(!ChainDelegators.empty());
+ BuildSearchTable();
}
- ui32 TAllChains::GetEndBlocks() const {
- return SizeToBlocks(MaxBlobInBytes);
+ inline ui32 TAllChains::SizeToBlocks(ui32 size) const {
+ ui32 sizeInBlocks = size / AppendBlockSize;
+ sizeInBlocks += !(sizeInBlocks * AppendBlockSize == size);
+ return sizeInBlocks;
}
- void TAllChains::FinishRecovery() {
- ui32 prevSlotSize = 0;
- for (const TChain& chain : Chains) {
- Y_ABORT_UNLESS(prevSlotSize < chain.SlotSize);
- prevSlotSize = chain.SlotSize;
- }
- BuildSearchTable();
+ inline ui32 TAllChains::GetEndBlocks() const {
+ ui32 endBlocks = MaxBlobInBytes / AppendBlockSize;
+ endBlocks += !(endBlocks * AppendBlockSize == MaxBlobInBytes);
+ return endBlocks;
}
////////////////////////////////////////////////////////////////////////////
@@ -669,6 +701,7 @@ namespace NKikimr {
ui32 chunkSize,
ui32 appendBlockSize,
ui32 minHugeBlobInBytes,
+ ui32 oldMinHugeBlobSizeInBytes,
ui32 mileStoneBlobInBytes,
ui32 maxBlobInBytes,
ui32 overhead,
@@ -676,41 +709,42 @@ namespace NKikimr {
: VDiskLogPrefix(vdiskLogPrefix)
, FreeChunksReservation(freeChunksReservation)
, FreeChunks()
- , Chains(vdiskLogPrefix, chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
- maxBlobInBytes, overhead)
+ , Chains(vdiskLogPrefix, chunkSize, appendBlockSize, minHugeBlobInBytes, oldMinHugeBlobSizeInBytes,
+ mileStoneBlobInBytes, maxBlobInBytes, overhead)
{}
//////////////////////////////////////////////////////////////////////////////////////////
// THeap: main functions
//////////////////////////////////////////////////////////////////////////////////////////
- THugeSlot THeap::ConvertDiskPartToHugeSlot(const TDiskPart& addr) const {
- const TChain *chain = Chains.GetChain(addr.Size);
- Y_ABORT_UNLESS(chain);
- return chain->Convert(chain->Convert(addr));
+ THugeSlot THeap::ConvertDiskPartToHugeSlot(const TDiskPart &addr) const {
+ const TChainDelegator *chainD = Chains.GetChain(addr.Size);
+ Y_VERIFY_S(chainD && (addr.Offset / chainD->SlotSize * chainD->SlotSize == addr.Offset), VDiskLogPrefix
+ << "chainD# " << (chainD ? chainD->ToString() : "nullptr") << " addr# " << addr.ToString());
+ return THugeSlot(addr.ChunkIdx, addr.Offset, chainD->SlotSize);
}
bool THeap::Allocate(ui32 size, THugeSlot *hugeSlot, ui32 *slotSize) {
- TChain *chain = Chains.GetChain(size);
- Y_VERIFY_S(chain, VDiskLogPrefix << "size# " << size << " Heap# " << ToString());
- *slotSize = chain->SlotSize;
+ TChainDelegator *chainD = Chains.GetChain(size);
+ Y_VERIFY_S(chainD, VDiskLogPrefix << "size# " << size << " Heap# " << ToString());
+ *slotSize = chainD->SlotSize;
NPrivate::TChunkSlot id;
- if (!chain->Allocate(&id)) { // no available slot in free space of the chain
+ if (!chainD->ChainPtr->Allocate(&id)) { // no available slot in free space of the chain
if (FreeChunks.empty()) { // no free chunks left for reuse -- request a new chunk
return false;
}
- chain->Allocate(&id, GetChunkIdFromFreeChunks()); // reuse free chunk for this chain
+ chainD->ChainPtr->Allocate(&id, GetChunkIdFromFreeChunks()); // reuse free chunk for this chain
}
- *hugeSlot = chain->Convert(id);
+ *hugeSlot = chainD->Convert(id);
return true;
}
TFreeRes THeap::Free(const TDiskPart &addr) {
ui32 size = addr.Size;
- TChain *chain = Chains.GetChain(size);
- Y_ABORT_UNLESS(chain);
+ TChainDelegator *chainD = Chains.GetChain(size);
+ Y_ABORT_UNLESS(chainD);
- TFreeRes res = chain->Free(chain->Convert(addr));
+ TFreeRes res = chainD->ChainPtr->Free(chainD->Convert(addr));
if (res.ChunkId) {
PutChunkIdToFreeChunks(res.ChunkId);
}
@@ -732,19 +766,13 @@ namespace NKikimr {
}
bool THeap::LockChunkForAllocation(ui32 chunkId, ui32 slotSize) {
- TChain *chain = Chains.GetChain(slotSize);
- Y_ABORT_UNLESS(chain);
- return chain->LockChunkForAllocation(chunkId);
+ TChainDelegator *cd = Chains.GetChain(slotSize);
+ return cd->ChainPtr->LockChunkForAllocation(chunkId);
}
void THeap::UnlockChunk(ui32 chunkId, ui32 slotSize) {
- TChain *chain = Chains.GetChain(slotSize);
- Y_ABORT_UNLESS(chain);
- chain->UnlockChunk(chunkId);
- }
-
- void THeap::FinishRecovery() {
- Chains.FinishRecovery();
+ TChainDelegator *cd = Chains.GetChain(slotSize);
+ cd->ChainPtr->UnlockChunk(chunkId);
}
THeapStat THeap::GetStat() const {
@@ -760,11 +788,11 @@ namespace NKikimr {
void THeap::RecoveryModeAllocate(const TDiskPart &addr) {
ui32 size = addr.Size;
- TChain *chain = Chains.GetChain(size);
- Y_VERIFY_S(chain, VDiskLogPrefix << "State# " << ToString());
+ TChainDelegator *chainD = Chains.GetChain(size);
+ Y_VERIFY_S(chainD, VDiskLogPrefix << "State# " << ToString());
- NPrivate::TChunkSlot id(chain->Convert(addr));
- bool allocated = chain->RecoveryModeAllocate(id);
+ NPrivate::TChunkSlot id(chainD->Convert(addr));
+ bool allocated = chainD->ChainPtr->RecoveryModeAllocate(id);
if (allocated) {
return;
} else {
@@ -772,7 +800,7 @@ namespace NKikimr {
TFreeChunks::iterator it = FreeChunks.find(chunkId);
Y_VERIFY_S(it != FreeChunks.end(), VDiskLogPrefix << "addr# " << addr.ToString() << " State# " << ToString());
FreeChunks.erase(it);
- chain->RecoveryModeAllocate(id, chunkId, false);
+ chainD->ChainPtr->RecoveryModeAllocate(id, chunkId, false);
}
}
@@ -789,9 +817,9 @@ namespace NKikimr {
}
bool THeap::ReleaseSlot(THugeSlot slot) {
- TChain* const chain = Chains.GetChain(slot.GetSize());
+ TChainDelegator* const chain = Chains.GetChain(slot.GetSize());
Y_VERIFY_S(chain, VDiskLogPrefix << "State# " << ToString() << " slot# " << slot.ToString());
- if (TFreeRes res = chain->Free(chain->Convert(slot)); res.ChunkId) {
+ if (TFreeRes res = chain->ChainPtr->Free(chain->Convert(slot)); res.ChunkId) {
PutChunkIdToFreeChunks(res.ChunkId);
return res.InLockedChunks;
}
@@ -799,12 +827,12 @@ namespace NKikimr {
}
void THeap::OccupySlot(THugeSlot slot, bool inLockedChunks) {
- TChain* const chain = Chains.GetChain(slot.GetSize());
+ TChainDelegator* const chain = Chains.GetChain(slot.GetSize());
Y_VERIFY_S(chain, VDiskLogPrefix << "State# " << ToString() << " slot# " << slot.ToString());
- if (!chain->RecoveryModeAllocate(chain->Convert(slot))) {
+ if (!chain->ChainPtr->RecoveryModeAllocate(chain->Convert(slot))) {
const size_t numErased = FreeChunks.erase(slot.GetChunkId());
Y_VERIFY_S(numErased, VDiskLogPrefix << "State# " << ToString() << " slot# " << slot.ToString());
- chain->RecoveryModeAllocate(chain->Convert(slot), slot.GetChunkId(), inLockedChunks);
+ chain->ChainPtr->RecoveryModeAllocate(chain->Convert(slot), slot.GetChunkId(), inLockedChunks);
}
}
@@ -871,8 +899,9 @@ namespace NKikimr {
TString THeap::ToString() const {
TStringStream str;
- str << "FreeChunks# " << FormatList(FreeChunks)
- << " Chains# {" << Chains.ToString() << '}';
+ str << "FreeChunks: ";
+ str << FormatList(FreeChunks);
+ str << " CHAINS: " << Chains.ToString();
return str.Str();
}
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h
index afdfe26959..6fbae987e4 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.h
@@ -70,7 +70,7 @@ namespace NKikimr {
TChainLayoutBuilder(ui32 left, ui32 milestone, ui32 right, ui32 overhead);
const TVector<TSeg> &GetLayout() const { return Layout; }
- const TSeg &GetMilestoneSegment() const { return Layout.at(MilestoneId); }
+ const TSeg &GetMilestoneSegment() const { return Layout.at(MilesoneId); }
TString ToString(ui32 appendBlockSize = 0) const;
void Output(IOutputStream &str, ui32 appendBlockSize = 0) const;
@@ -81,7 +81,7 @@ namespace NKikimr {
TVector<TSeg> Layout;
// An index in Layout vector, where milestone segment starts
- size_t MilestoneId = Max<size_t>();
+ size_t MilesoneId = Max<size_t>();
};
} // NPrivate
@@ -90,43 +90,29 @@ namespace NKikimr {
// TChain
// It manages all slots of some fixed size.
////////////////////////////////////////////////////////////////////////////
- class TChain {
+ class TChain : public TThrRefBase {
using TChunkID = ui32;
using TFreeSpace = TMap<TChunkID, TMask>;
static constexpr ui32 MaxNumberOfSlots = 32768; // it's not a good idea to have more slots than this
- /*const*/ TString VDiskLogPrefix;
- /*const*/ TMask ConstMask; // mask of 'all slots are free'
+ const TString VDiskLogPrefix;
+ const ui32 SlotsInChunk;
+ const TMask ConstMask; // mask of 'all slots are free'
TFreeSpace FreeSpace;
TFreeSpace LockedChunks;
ui32 AllocatedSlots = 0;
ui32 FreeSlotsInFreeSpace = 0;
public:
- /*const*/ ui32 SlotsInChunk;
- /*const*/ ui32 SlotSize;
-
- public:
static TMask BuildConstMask(const TString &prefix, ui32 slotsInChunk);
public:
- TChain(TString vdiskLogPrefix, ui32 slotsInChunk, ui32 slotSize)
- : VDiskLogPrefix(std::move(vdiskLogPrefix))
- , ConstMask(BuildConstMask(vdiskLogPrefix, slotsInChunk))
+ TChain(const TString &vdiskLogPrefix, const ui32 slotsInChunk)
+ : VDiskLogPrefix(vdiskLogPrefix)
, SlotsInChunk(slotsInChunk)
- , SlotSize(slotSize)
+ , ConstMask(BuildConstMask(vdiskLogPrefix, slotsInChunk))
{}
- TChain(TChain&&) = default;
- TChain(const TChain&) = delete;
-
- TChain& operator=(TChain&&) = default;
- TChain& operator=(const TChain&) = delete;
-
- THugeSlot Convert(const NPrivate::TChunkSlot& id) const;
- NPrivate::TChunkSlot Convert(const TDiskPart& addr) const;
- NPrivate::TChunkSlot Convert(const THugeSlot& slot) const;
-
// returns true if allocated, false -- if no free slots
bool Allocate(NPrivate::TChunkSlot *id);
// allocate id, but we know that this chain doesn't have free slots, so add a chunk to it
@@ -140,49 +126,71 @@ namespace NKikimr {
bool RecoveryModeAllocate(const NPrivate::TChunkSlot &id);
void RecoveryModeAllocate(const NPrivate::TChunkSlot &id, TChunkID chunkId, bool inLockedChunks);
void Save(IOutputStream *s) const;
+ void Load(IInputStream *s);
bool HaveBeenUsed() const;
TString ToString() const;
void RenderHtml(IOutputStream &str) const;
- void RenderHtmlForUsage(IOutputStream &str) const;
+ ui32 GetAllocatedSlots() const;
void GetOwnedChunks(TSet<TChunkIdx>& chunks) const;
+ };
- static TChain Load(IInputStream *s, TString vdiskLogPrefix, ui32 appendBlockSize, ui32 blocksInChunk,
- std::span<TChain> chains, bool *compatible);
-
- template<typename T>
- void ForEachFreeSpaceChunk(T&& callback) const {
- auto freeIt = FreeSpace.begin();
- const auto freeEnd = FreeSpace.end();
- auto lockedIt = LockedChunks.begin();
- const auto lockedEnd = LockedChunks.end();
- while (freeIt != freeEnd || lockedIt != lockedEnd) {
- if (lockedIt == lockedEnd || freeIt->first < lockedIt->first) {
- std::invoke(callback, *freeIt++);
- } else if (freeIt == freeEnd || lockedIt->first < freeIt->first) {
- std::invoke(callback, *lockedIt++);
- } else {
- Y_ABORT("intersecting sets of keys for FreeSpace and LockedChunks");
- }
- }
- }
+ using TChainPtr = TIntrusivePtr<TChain>;
+
+ ////////////////////////////////////////////////////////////////////////////
+ // TChainDelegator
+ ////////////////////////////////////////////////////////////////////////////
+ struct TChainDelegator {
+ TString VDiskLogPrefix;
+ ui32 Blocks;
+ ui32 ShiftInBlocks;
+ ui32 SlotsInChunk;
+ ui32 SlotSize;
+ TChainPtr ChainPtr;
+
+ TChainDelegator(const TString &vdiskLogPrefix,
+ ui32 valBlocks,
+ ui32 shiftBlocks,
+ ui32 chunkSize,
+ ui32 appendBlockSize);
+ TChainDelegator(TChainDelegator &&) = default;
+ TChainDelegator &operator =(TChainDelegator &&) = default;
+ TChainDelegator(const TChainDelegator &) = delete;
+ TChainDelegator &operator =(const TChainDelegator &) = delete;
+ THugeSlot Convert(const NPrivate::TChunkSlot &id) const;
+ NPrivate::TChunkSlot Convert(const TDiskPart &addr) const;
+ NPrivate::TChunkSlot Convert(const THugeSlot &slot) const;
+ void Save(IOutputStream *s) const;
+ void Load(IInputStream *s);
+ bool HaveBeenUsed() const;
+ TString ToString() const;
+ void GetOwnedChunks(TSet<TChunkIdx>& chunks) const;
+ void RenderHtml(IOutputStream &str) const;
+ void RenderHtmlForUsage(IOutputStream &str) const;
};
+
////////////////////////////////////////////////////////////////////////////
// TAllChains
////////////////////////////////////////////////////////////////////////////
class TAllChains {
public:
+ using TAllChainDelegators = TVector<TChainDelegator>;
+ using TSearchTable = TVector<TChainDelegator*>;
+
TAllChains(const TString &vdiskLogPrefix,
ui32 chunkSize,
ui32 appendBlockSize,
ui32 minHugeBlobInBytes,
+ ui32 oldMinHugeBlobSizeInBytes,
ui32 milestoneBlobInBytes,
ui32 maxBlobInBytes,
ui32 overhead);
// return a pointer to corresponding chain delegator by object byte size
- TChain *GetChain(ui32 size);
- const TChain *GetChain(ui32 size) const;
+ TChainDelegator *GetChain(ui32 size);
+ const TChainDelegator *GetChain(ui32 size) const;
THeapStat GetStat() const;
+ void PrintOutChains(IOutputStream &str) const;
+ void PrintOutSearchTable(IOutputStream &str) const;
void Save(IOutputStream *s) const;
void Load(IInputStream *s);
void GetOwnedChunks(TSet<TChunkIdx>& chunks) const;
@@ -191,28 +199,39 @@ namespace NKikimr {
void RenderHtmlForUsage(IOutputStream &str) const;
// for testing purposes
TVector<NPrivate::TChainLayoutBuilder::TSeg> GetLayout() const;
+ // returns (ChainsSize, SearchTableSize)
+ std::pair<ui32, ui32> GetTablesSize() const {
+ return std::pair<ui32, ui32>(ChainDelegators.size(), SearchTable.size());
+ }
// Builds a map of BlobSize -> THugeSlotsMap::TSlotInfo for THugeBlobCtx
std::shared_ptr<THugeSlotsMap> BuildHugeSlotsMap() const;
- void FinishRecovery();
-
private:
- void BuildChains();
+
+ TAllChainDelegators BuildChains(ui32 minHugeBlobInBytes) const;
void BuildSearchTable();
+ void BuildLayout();
inline ui32 SizeToBlocks(ui32 size) const;
inline ui32 GetEndBlocks() const;
+ enum class EStartMode {
+ Empty = 1,
+ Loaded = 2,
+ Migrated = 3,
+ };
+
const TString VDiskLogPrefix;
const ui32 ChunkSize;
const ui32 AppendBlockSize;
const ui32 MinHugeBlobInBytes;
+ const ui32 OldMinHugeBlobSizeInBytes;
const ui32 MilestoneBlobInBytes;
const ui32 MaxBlobInBytes;
const ui32 Overhead;
- const ui32 MinHugeBlobInBlocks;
- THashSet<ui32> DeserializedSlotSizes; // a set of SlotSize values of recovered stream
- std::vector<TChain> Chains;
- std::vector<ui16> SearchTable; // (NumFullBlocks - 1) -> Chain index
+ EStartMode StartMode = EStartMode::Empty;
+ ui32 FirstLoadedSlotSize = 0;
+ TAllChainDelegators ChainDelegators;
+ TSearchTable SearchTable;
};
@@ -236,6 +255,7 @@ namespace NKikimr {
ui32 appendBlockSize,
// min size of the huge blob
ui32 minHugeBlobInBytes,
+ ui32 oldMinHugeBlobSizeInBytes,
// fixed point to calculate layout (for backward compatibility)
ui32 mileStoneBlobInBytes,
// max size of the blob
@@ -246,13 +266,13 @@ namespace NKikimr {
ui32 SlotNumberOfThisSize(ui32 size) const {
- const TChain *chain = Chains.GetChain(size);
- return chain ? chain->SlotsInChunk : 0;
+ const TChainDelegator *chainD = Chains.GetChain(size);
+ return chainD ? chainD->SlotsInChunk : 0;
}
ui32 SlotSizeOfThisSize(ui32 size) const {
- const TChain *chain = Chains.GetChain(size);
- return chain ? chain->SlotSize : 0;
+ const TChainDelegator *chainD = Chains.GetChain(size);
+ return chainD ? chainD->SlotSize : 0;
}
// Builds a map of BlobSize -> THugeSlotsMap::TSlotInfo for THugeBlobCtx
@@ -282,7 +302,6 @@ namespace NKikimr {
void RecoveryModeRemoveChunks(const TVector<ui32> &chunkIds);
bool ReleaseSlot(THugeSlot slot);
void OccupySlot(THugeSlot slot, bool inLockedChunks);
- void FinishRecovery();
//////////////////////////////////////////////////////////////////////////////////////////
// Serialize/Parse/Check
@@ -298,6 +317,10 @@ namespace NKikimr {
void RenderHtml(IOutputStream &str) const;
TString ToString() const;
+ void PrintOutSearchTable(IOutputStream &str) {
+ Chains.PrintOutSearchTable(str);
+ }
+
private:
inline ui32 GetChunkIdFromFreeChunks();
inline void PutChunkIdToFreeChunks(ui32 chunkId);
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp
index 865bf38513..df27d118e9 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ctx_ut.cpp
@@ -26,6 +26,7 @@ namespace NKikimr {
Contexts.GetVCtx(),
ChunkSize,
AppendBlockSize,
+ AppendBlockSize,
cfg.MinHugeBlobInBytes,
cfg.MilestoneHugeBlobInBytes,
cfg.MaxLogoBlobDataSize,
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp
index 0edd937d24..5de29e6f46 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap_ut.cpp
@@ -79,7 +79,7 @@ namespace NKikimr {
}
void AllocFreeOneChunk(ui32 slotsInChunk) {
- TChain chain("vdisk", slotsInChunk, 1);
+ TChain chain("vdisk", slotsInChunk);
TVector<NPrivate::TChunkSlot> arr;
AllocateScenaryOneChunk(chain, arr, slotsInChunk);
FreeScenaryOneChunk(chain, arr, slotsInChunk);
@@ -156,7 +156,7 @@ namespace NKikimr {
}
void AllocFreeAlloc(ui32 slotsInChunk) {
- TChain chain("vdisk", slotsInChunk, 1);
+ TChain chain("vdisk", slotsInChunk);
TVector<NPrivate::TChunkSlot> arr;
TVector<ui32> chunks;
@@ -171,14 +171,17 @@ namespace NKikimr {
TStringStream serialized;
- TChain chain("vdisk", slotsInChunk, 1);
- PreliminaryAllocate(24, chain, arr);
- FreeChunksScenary(chain, arr, chunks);
- chain.Save(&serialized);
-
- TStringInput str(serialized.Str());
- bool comp = true;
- TChain chain2 = TChain::Load(&str, "vdisk", 1 /*appendBlockSize*/, slotsInChunk, {&chain, &chain + 1}, &comp);
+ {
+ TChain chain("vdisk", slotsInChunk);
+ PreliminaryAllocate(24, chain, arr);
+ FreeChunksScenary(chain, arr, chunks);
+ chain.Save(&serialized);
+ }
+ {
+ TChain chain("vdisk", slotsInChunk);
+ TStringInput str(serialized.Str());
+ chain.Load(&str);
+ }
}
Y_UNIT_TEST(AllocFreeAllocTest) {
@@ -245,6 +248,38 @@ namespace NKikimr {
}
}
+ Y_UNIT_TEST_SUITE(TBlobStorageHullHugeLayout) {
+
+ Y_UNIT_TEST(TestOldAppendBlockSize) {
+ TAllChains all("vdisk", 134274560, 56896, 512 << 10, 512 << 10, 512 << 10, 10 << 20, 8);
+ all.PrintOutChains(STR);
+ all.PrintOutSearchTable(STR);
+ std::pair<ui32, ui32> p = all.GetTablesSize();
+ TVector<NPrivate::TChainLayoutBuilder::TSeg> canonical = {
+ {9, 10}, {10, 11}, {11, 12}, {12, 13}, {13, 14}, {14, 15}, {15, 16}, {16, 18}, {18, 20},
+ {20, 22}, {22, 24}, {24, 27}, {27, 30}, {30, 33}, {33, 37}, {37, 41}, {41, 46}, {46, 51},
+ {51, 57}, {57, 64}, {64, 72}, {72, 81}, {81, 91}, {91, 102}, {102, 114}, {114, 128},
+ {128, 144}, {144, 162}, {162, 182}, {182, 204}
+ };
+ UNIT_ASSERT_EQUAL(all.GetLayout(), canonical);
+ UNIT_ASSERT_EQUAL(p, (std::pair<ui32, ui32>(30, 186)));
+ }
+
+ Y_UNIT_TEST(TestNewAppendBlockSize) {
+ TAllChains all("vdisk", 134274560, 4064, 512 << 10, 512 << 10, 512 << 10, 10 << 20, 8);
+ all.PrintOutChains(STR);
+ all.PrintOutSearchTable(STR);
+ TVector<NPrivate::TChainLayoutBuilder::TSeg> canonical = {
+ {129, 145}, {145, 163}, {163, 183}, {183, 205}, {205, 230}, {230, 258}, {258, 290},
+ {290, 326}, {326, 366}, {366, 411}, {411, 462}, {462, 519}, {519, 583}, {583, 655},
+ {655, 736}, {736, 828}, {828, 931}, {931, 1047}, {1047, 1177}, {1177, 1324},
+ {1324, 1489}, {1489, 1675}, {1675, 1884}, {1884, 2119}, {2119, 2383}, {2383, 2680}
+ };
+ UNIT_ASSERT_EQUAL(all.GetLayout(), canonical);
+ }
+ }
+
+
Y_UNIT_TEST_SUITE(TBlobStorageHullHugeHeap) {
Y_UNIT_TEST(AllocateAllFromOneChunk) {
@@ -255,9 +290,8 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
- heap.FinishRecovery();
ui32 hugeBlobSize = 6u << 20u;
heap.AddChunk(5);
@@ -280,10 +314,9 @@ namespace NKikimr {
// just serialize/deserialize
TString serialized = heap.Serialize();
- THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
newHeap.ParseFromString(serialized);
- newHeap.FinishRecovery();
}
void AllocateScenary(THeap &heap, ui32 hugeBlobSize, TVector<THugeSlot> &arr) {
@@ -324,9 +357,8 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
- heap.FinishRecovery();
TVector<THugeSlot> arr;
AllocateScenary(heap, 6u << 20u, arr);
@@ -340,21 +372,16 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
- heap.FinishRecovery();
TVector<THugeSlot> arr;
AllocateScenary(heap, 6u << 20u, arr);
- TString heap1 = heap.ToString();
TString serialized = heap.Serialize();
UNIT_ASSERT(THeap::CheckEntryPoint(serialized));
- THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap newHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
newHeap.ParseFromString(serialized);
- newHeap.FinishRecovery();
- TString heap2 = newHeap.ToString();
- UNIT_ASSERT_VALUES_EQUAL(heap1, heap2);
FreeScenary(newHeap, arr);
}
@@ -365,9 +392,8 @@ namespace NKikimr {
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
- heap.FinishRecovery();
heap.RecoveryModeAddChunk(2);
heap.RecoveryModeAddChunk(34);
@@ -392,17 +418,17 @@ namespace NKikimr {
ui32 chunkSize = 134274560u;
ui32 appendBlockSize = 56896u;
ui32 minHugeBlobInBytes = appendBlockSize;
+ ui32 minREALHugeBlobInBytes = minHugeBlobInBytes / appendBlockSize * appendBlockSize + 1;
ui32 maxBlobInBytes = MaxVDiskBlobSize;
ui32 overhead = 8u;
ui32 freeChunksReservation = 1;
- THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes,
+ THeap heap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, minHugeBlobInBytes, minHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
- heap.FinishRecovery();
THugeSlot hugeSlot;
ui32 slotSize;
bool res = false;
- res = heap.Allocate(minHugeBlobInBytes, &hugeSlot, &slotSize);
+ res = heap.Allocate(minREALHugeBlobInBytes, &hugeSlot, &slotSize);
UNIT_ASSERT_EQUAL(res, false); // no chunks
res = heap.Allocate(maxBlobInBytes, &hugeSlot, &slotSize);
UNIT_ASSERT_EQUAL(res, false); // no chunks
@@ -413,34 +439,52 @@ namespace NKikimr {
RollbackFrom_New_To_Old,
};
- Y_UNIT_TEST(WriteRestore) {
+ void Write_SaveEntryPoint_Restart(EWrite_SaveEntryPoint_Restart mode) {
ui32 chunkSize = 134274560u;
ui32 appendBlockSize = 4064u;
ui32 minHugeBlobInBytes = appendBlockSize;
+ ui32 oldMinHugeBlobInBytes = 64u << 10u;
ui32 mileStoneBlobInBytes = 512u << 10u;
ui32 maxBlobInBytes = 10u << 20u;
ui32 overhead = 8;
ui32 freeChunksReservation = 0;
- THeap oldHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ ui32 fromMin = 0;
+ ui32 toMin = 0;
+ switch (mode) {
+ case EWrite_SaveEntryPoint_Restart::MigrateFrom_Old_To_New:
+ fromMin = oldMinHugeBlobInBytes;
+ toMin = minHugeBlobInBytes;
+ break;
+ case EWrite_SaveEntryPoint_Restart::RollbackFrom_New_To_Old:
+ fromMin = minHugeBlobInBytes;
+ toMin = oldMinHugeBlobInBytes;
+ break;
+ }
+ THeap oldHeap("vdisk", chunkSize, appendBlockSize, oldMinHugeBlobInBytes, oldMinHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
- oldHeap.FinishRecovery();
- THeap fromHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap fromHeap("vdisk", chunkSize, appendBlockSize, fromMin, oldMinHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
fromHeap.ParseFromString(oldHeap.Serialize());
- fromHeap.FinishRecovery();
TVector<THugeSlot> arr;
AllocateScenary(fromHeap, 6u << 20u, arr);
TString serialized = fromHeap.Serialize();
UNIT_ASSERT(THeap::CheckEntryPoint(serialized));
- THeap toHeap("vdisk", chunkSize, appendBlockSize, minHugeBlobInBytes, mileStoneBlobInBytes,
+ THeap toHeap("vdisk", chunkSize, appendBlockSize, toMin, oldMinHugeBlobInBytes, mileStoneBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation);
toHeap.ParseFromString(serialized);
- toHeap.FinishRecovery();
FreeScenary(toHeap, arr);
}
+
+ Y_UNIT_TEST(MigrateFrom_Old_To_New) {
+ Write_SaveEntryPoint_Restart(EWrite_SaveEntryPoint_Restart::MigrateFrom_Old_To_New);
+ }
+
+ Y_UNIT_TEST(RollbackFrom_New_To_Old) {
+ Write_SaveEntryPoint_Restart(EWrite_SaveEntryPoint_Restart::RollbackFrom_New_To_Old);
+ }
}
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp
index f5aa9d561f..898271ef6d 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.cpp
@@ -64,6 +64,7 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
+ const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -72,11 +73,10 @@ namespace NKikimr {
: VCtx(std::move(vctx))
, LogPos(THullHugeRecoveryLogPos::Default())
, Heap(new NHuge::THeap(VCtx->VDiskLogPrefix, chunkSize, appendBlockSize,
- minHugeBlobInBytes, milestoneHugeBlobInBytes,
+ minHugeBlobInBytes, oldMinHugeBlobInBytes, milestoneHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation))
, Guid(TAppData::RandomProvider->GenRand64())
{
- Heap->FinishRecovery();
logFunc(VDISKP(VCtx->VDiskLogPrefix,
"Recovery started (guid# %" PRIu64 " entryLsn# null): State# %s",
Guid, ToString().data()));
@@ -86,6 +86,7 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
+ const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -96,13 +97,12 @@ namespace NKikimr {
: VCtx(std::move(vctx))
, LogPos(THullHugeRecoveryLogPos::Default())
, Heap(new NHuge::THeap(VCtx->VDiskLogPrefix, chunkSize, appendBlockSize,
- minHugeBlobInBytes, milestoneHugeBlobInBytes,
+ minHugeBlobInBytes, oldMinHugeBlobInBytes, milestoneHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation))
, Guid(TAppData::RandomProvider->GenRand64())
, PersistentLsn(entryPointLsn)
{
ParseFromString(entryPointData);
- Heap->FinishRecovery();
Y_ABORT_UNLESS(entryPointLsn == LogPos.EntryPointLsn);
logFunc(VDISKP(VCtx->VDiskLogPrefix,
"Recovery started (guid# %" PRIu64 " entryLsn# %" PRIu64 "): State# %s",
@@ -113,6 +113,7 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
+ const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -123,13 +124,12 @@ namespace NKikimr {
: VCtx(std::move(vctx))
, LogPos(THullHugeRecoveryLogPos::Default())
, Heap(new NHuge::THeap(VCtx->VDiskLogPrefix, chunkSize, appendBlockSize,
- minHugeBlobInBytes, milestoneHugeBlobInBytes,
+ minHugeBlobInBytes, oldMinHugeBlobInBytes, milestoneHugeBlobInBytes,
maxBlobInBytes, overhead, freeChunksReservation))
, Guid(TAppData::RandomProvider->GenRand64())
, PersistentLsn(entryPointLsn)
{
ParseFromArray(entryPointData.GetData(), entryPointData.GetSize());
- Heap->FinishRecovery();
Y_ABORT_UNLESS(entryPointLsn == LogPos.EntryPointLsn);
logFunc(VDISKP(VCtx->VDiskLogPrefix,
"Recovery started (guid# %" PRIu64 " entryLsn# %" PRIu64 "): State# %s",
diff --git a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h
index cea53fbe31..ffac5bdb0c 100644
--- a/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h
+++ b/ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugerecovery.h
@@ -92,6 +92,7 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
+ const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -101,6 +102,7 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
+ const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -112,6 +114,7 @@ namespace NKikimr {
const ui32 chunkSize,
const ui32 appendBlockSize,
const ui32 minHugeBlobInBytes,
+ const ui32 oldMinHugeBlobInBytes,
const ui32 milestoneHugeBlobInBytes,
const ui32 maxBlobInBytes,
const ui32 overhead,
@@ -130,7 +133,7 @@ namespace NKikimr {
static bool CheckEntryPoint(TContiguousSpan data);
TString ToString() const;
void RenderHtml(IOutputStream &str) const;
- ui32 GetMinHugeBlobInBytes() const;
+ ui32 GetMinREALHugeBlobInBytes() const;
ui64 FirstLsnToKeep(ui64 minInFlightLsn = Max<ui64>()) const;
TString FirstLsnToKeepDecomposed() const;
bool WouldNewEntryPointAdvanceLog(ui64 freeUpToLsn, ui64 minInFlightLsn, ui32 itemsAfterCommit) const;
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp
index cb8bff996e..8268554c68 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.cpp
@@ -38,13 +38,13 @@ namespace NKikimr {
TDelayedResponses DelayedResponses;
bool AllowGarbageCollection = false;
THugeBlobCtxPtr HugeBlobCtx;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
TFields(TIntrusivePtr<THullDs> hullDs,
TIntrusivePtr<TLsnMngr> &&lsnMngr,
TPDiskCtxPtr &&pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
const TActorId skeletonId,
bool runHandoff,
TActorSystem *as,
@@ -60,7 +60,7 @@ namespace NKikimr {
, ActorSystem(as)
, BarrierValidation(barrierValidation)
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
{}
void CutRecoveryLog(const TActorContext &ctx, std::unique_ptr<NPDisk::TEvCutLog> msg) {
@@ -83,7 +83,7 @@ namespace NKikimr {
TIntrusivePtr<TLsnMngr> lsnMngr,
TPDiskCtxPtr pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
const TActorId skeletonId,
bool runHandoff,
THullDbRecovery &&uncond,
@@ -92,7 +92,7 @@ namespace NKikimr {
TActorId hugeKeeperId)
: THullDbRecovery(std::move(uncond))
, Fields(std::make_unique<TFields>(HullDs, std::move(lsnMngr), std::move(pdiskCtx), std::move(hugeBlobCtx),
- minHugeBlobInBytes, skeletonId, runHandoff, as, barrierValidation, hugeKeeperId))
+ minREALHugeBlobInBytes, skeletonId, runHandoff, as, barrierValidation, hugeKeeperId))
{}
THull::~THull() = default;
@@ -130,7 +130,7 @@ namespace NKikimr {
Fields->SetLogNotifierActorId(logNotifierAid);
// actor for LogoBlobs DB
HullDs->LogoBlobs->LIActor = ctx.RegisterWithSameMailbox(CreateLogoBlobsActor(config, HullDs, hullLogCtx,
- Fields->HugeBlobCtx, Fields->MinHugeBlobInBytes, loggerId, Fields->LogoBlobsRunTimeCtx, syncLogFirstLsnToKeep));
+ Fields->HugeBlobCtx, Fields->MinREALHugeBlobInBytes, loggerId, Fields->LogoBlobsRunTimeCtx, syncLogFirstLsnToKeep));
activeActors.Insert(HullDs->LogoBlobs->LIActor, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
// actor for Blocks DB
HullDs->Blocks->LIActor = ctx.RegisterWithSameMailbox(CreateBlocksActor(config, HullDs, hullLogCtx, loggerId,
@@ -711,14 +711,14 @@ namespace NKikimr {
ctx.Send(HullDs->Barriers->LIActor, new TEvPermitGarbageCollection);
}
- void THull::ApplyHugeBlobSize(ui32 minHugeBlobInBytes, const TActorContext& ctx) {
- Fields->MinHugeBlobInBytes = minHugeBlobInBytes;
- ctx.Send(HullDs->LogoBlobs->LIActor, new TEvMinHugeBlobSizeUpdate(minHugeBlobInBytes));
+ void THull::ApplyHugeBlobSize(ui32 minREALHugeBlobInBytes, const TActorContext& ctx) {
+ Fields->MinREALHugeBlobInBytes = minREALHugeBlobInBytes;
+ ctx.Send(HullDs->LogoBlobs->LIActor, new TEvMinHugeBlobSizeUpdate(minREALHugeBlobInBytes));
}
void THull::CompactFreshLogoBlobsIfRequired(const TActorContext& ctx) {
CompactFreshSegmentIfRequired<TKeyLogoBlob, TMemRecLogoBlob>(HullDs, Fields->HugeBlobCtx,
- Fields->MinHugeBlobInBytes, Fields->LogoBlobsRunTimeCtx, ctx, false, Fields->AllowGarbageCollection);
+ Fields->MinREALHugeBlobInBytes, Fields->LogoBlobsRunTimeCtx, ctx, false, Fields->AllowGarbageCollection);
}
} // NKikimr
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h
index 0e1b8cd98a..9d43370da5 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hull.h
@@ -62,7 +62,7 @@ namespace NKikimr {
TIntrusivePtr<TLsnMngr> lsnMngr,
TPDiskCtxPtr pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
const TActorId skeletonId,
bool runHandoff,
THullDbRecovery &&uncond,
@@ -215,7 +215,7 @@ namespace NKikimr {
void PermitGarbageCollection(const TActorContext& ctx);
- void ApplyHugeBlobSize(ui32 minHugeBlobInBytes, const TActorContext& ctx);
+ void ApplyHugeBlobSize(ui32 minREALHugeBlobInBytes, const TActorContext& ctx);
void CompactFreshLogoBlobsIfRequired(const TActorContext& ctx);
};
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp
index 9df6e15b26..1edb7cc955 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.cpp
@@ -78,7 +78,7 @@ namespace NKikimr {
void CompactFreshSegment(
TIntrusivePtr<THullDs> &hullDs,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
std::shared_ptr<TLevelIndexRunTimeCtx<TKey, TMemRec>> &rtCtx,
const TActorContext &ctx,
bool allowGarbageCollection)
@@ -107,7 +107,7 @@ namespace NKikimr {
ui64 firstLsn = freshSegment->GetFirstLsn();
ui64 lastLsn = freshSegment->GetLastLsn();
std::unique_ptr<TFreshCompaction> compaction(new TFreshCompaction(
- hullCtx, rtCtx, std::move(hugeBlobCtx), minHugeBlobInBytes, freshSegment, freshSegmentSnap,
+ hullCtx, rtCtx, std::move(hugeBlobCtx), minREALHugeBlobInBytes, freshSegment, freshSegmentSnap,
std::move(barriersSnap), std::move(levelSnap), mergeElementsApproximation, it, firstLsn, lastLsn,
TDuration::Max(), {}, allowGarbageCollection));
@@ -176,7 +176,7 @@ namespace NKikimr {
TInstant NextCompactionWakeup;
bool AllowGarbageCollection = false;
THugeBlobCtxPtr HugeBlobCtx;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
friend class TActorBootstrapped<TThis>;
@@ -233,7 +233,7 @@ namespace NKikimr {
void ScheduleCompaction(const TActorContext &ctx) {
// schedule fresh if required
- CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinHugeBlobInBytes, RTCtx, ctx,
+ CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinREALHugeBlobInBytes, RTCtx, ctx,
FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
if (!Config->BaseInfo.ReadOnly && !RunLevelCompactionSelector(ctx)) {
ScheduleCompactionWakeup(ctx);
@@ -261,7 +261,7 @@ namespace NKikimr {
it.SeekToFirst();
std::unique_ptr<TLevelCompaction> compaction(new TLevelCompaction(HullDs->HullCtx, RTCtx, HugeBlobCtx,
- MinHugeBlobInBytes, nullptr, nullptr, std::move(barriersSnap), std::move(levelSnap),
+ MinREALHugeBlobInBytes, nullptr, nullptr, std::move(barriersSnap), std::move(levelSnap),
mergeElementsApproximation, it, firstLsn, lastLsn, TDuration::Minutes(2), {}, AllowGarbageCollection));
NActors::TActorId actorId = RunInBatchPool(ctx, compaction.release());
ActiveActors.Insert(actorId, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
@@ -571,7 +571,7 @@ namespace NKikimr {
if (FullCompactionState.Enabled()) {
ScheduleCompaction(ctx);
} else {
- CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinHugeBlobInBytes, RTCtx,
+ CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinREALHugeBlobInBytes, RTCtx,
ctx, FullCompactionState.ForceFreshCompaction(RTCtx), AllowGarbageCollection);
}
break;
@@ -596,7 +596,7 @@ namespace NKikimr {
RTCtx->SetFreeUpToLsn(freeUpToLsn);
// we check if we need to start fresh compaction, FreeUpToLsn influence our decision
const bool freshCompStarted = CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx,
- MinHugeBlobInBytes, RTCtx, ctx, FullCompactionState.ForceFreshCompaction(RTCtx),
+ MinREALHugeBlobInBytes, RTCtx, ctx, FullCompactionState.ForceFreshCompaction(RTCtx),
AllowGarbageCollection);
// just for valid info output to the log
bool moveEntryPointStarted = false;
@@ -655,7 +655,7 @@ namespace NKikimr {
case E::FRESH_ONLY:
Y_ABORT_UNLESS(FreshOnlyCompactQ.empty() || FreshOnlyCompactQ.back().first <= confirmedLsn);
FreshOnlyCompactQ.emplace_back(confirmedLsn, ev);
- CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinHugeBlobInBytes, RTCtx, ctx,
+ CompactFreshSegmentIfRequired<TKey, TMemRec>(HullDs, HugeBlobCtx, MinREALHugeBlobInBytes, RTCtx, ctx,
true, AllowGarbageCollection); // ask for forced fresh compaction
break;
}
@@ -686,7 +686,7 @@ namespace NKikimr {
}
void Handle(TEvMinHugeBlobSizeUpdate::TPtr ev, const TActorContext& /*ctx*/) {
- MinHugeBlobInBytes = ev->Get()->MinHugeBlobInBytes;
+ MinREALHugeBlobInBytes = ev->Get()->MinREALHugeBlobInBytes;
}
STRICT_STFUNC(StateFunc,
@@ -714,7 +714,7 @@ namespace NKikimr {
TIntrusivePtr<THullDs> hullDs,
std::shared_ptr<THullLogCtx> hullLogCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TActorId loggerId,
std::shared_ptr<TRunTimeCtx> rtCtx,
std::shared_ptr<NSyncLog::TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep)
@@ -738,7 +738,7 @@ namespace NKikimr {
, ActiveActors(RTCtx->LevelIndex->ActorCtx->ActiveActors)
, LevelStat(HullDs->HullCtx->VCtx->VDiskCounters)
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
{}
};
@@ -747,12 +747,12 @@ namespace NKikimr {
TIntrusivePtr<THullDs> hullDs,
std::shared_ptr<THullLogCtx> hullLogCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TActorId loggerId,
std::shared_ptr<TLevelIndexRunTimeCtx<TKeyLogoBlob, TMemRecLogoBlob>> rtCtx,
std::shared_ptr<NSyncLog::TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep) {
return new TLevelIndexActor<TKeyLogoBlob, TMemRecLogoBlob>(config, hullDs, hullLogCtx, std::move(hugeBlobCtx),
- minHugeBlobInBytes, loggerId, rtCtx,syncLogFirstLsnToKeep);
+ minREALHugeBlobInBytes, loggerId, rtCtx,syncLogFirstLsnToKeep);
}
NActors::IActor* CreateBlocksActor(
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h
index c47fe75616..e740a9f47e 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullactor.h
@@ -84,7 +84,7 @@ namespace NKikimr {
void CompactFreshSegment(
TIntrusivePtr<THullDs> &hullDs,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
std::shared_ptr<TLevelIndexRunTimeCtx<TKey, TMemRec>> &rtCtx,
const TActorContext &ctx,
bool allowGarbageCollection);
@@ -93,7 +93,7 @@ namespace NKikimr {
bool CompactFreshSegmentIfRequired(
TIntrusivePtr<THullDs> &hullDs,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
std::shared_ptr<TLevelIndexRunTimeCtx<TKey, TMemRec>> &rtCtx,
const TActorContext &ctx,
bool force,
@@ -102,7 +102,7 @@ namespace NKikimr {
ui64 yardFreeUpToLsn = rtCtx->GetFreeUpToLsn();
bool compact = hullDs->HullCtx->FreshCompaction && rtCtx->LevelIndex->NeedsFreshCompaction(yardFreeUpToLsn, force);
if (compact) {
- CompactFreshSegment<TKey, TMemRec>(hullDs, std::move(hugeBlobCtx), minHugeBlobInBytes, rtCtx, ctx,
+ CompactFreshSegment<TKey, TMemRec>(hullDs, std::move(hugeBlobCtx), minREALHugeBlobInBytes, rtCtx, ctx,
allowGarbageCollection);
}
return compact;
@@ -116,7 +116,7 @@ namespace NKikimr {
TIntrusivePtr<THullDs> hullDs,
std::shared_ptr<THullLogCtx> hullLogCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TActorId loggerId,
std::shared_ptr<TLevelIndexRunTimeCtx<TKeyLogoBlob, TMemRecLogoBlob>> rtCtx,
std::shared_ptr<NSyncLog::TSyncLogFirstLsnToKeep> syncLogFirstLsnToKeep);
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h
index 67c50873f7..73ffcdc8c3 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompact.h
@@ -323,7 +323,7 @@ namespace NKikimr {
THullCompaction(THullCtxPtr hullCtx,
const std::shared_ptr<TLevelIndexRunTimeCtx> &rtCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TIntrusivePtr<TFreshSegment> freshSegment,
std::shared_ptr<TFreshSegmentSnapshot> freshSegmentSnap,
TBarriersSnapshot &&barriersSnap,
@@ -346,7 +346,7 @@ namespace NKikimr {
, Hmp(CreateHandoffMap<TKey, TMemRec>(HullCtx, rtCtx->RunHandoff, rtCtx->SkeletonId))
, Gcmp(CreateGcMap<TKey, TMemRec>(HullCtx, mergeElementsApproximation, allowGarbageCollection))
, It(it)
- , Worker(HullCtx, PDiskCtx, std::move(hugeBlobCtx), minHugeBlobInBytes, rtCtx->LevelIndex, it,
+ , Worker(HullCtx, PDiskCtx, std::move(hugeBlobCtx), minREALHugeBlobInBytes, rtCtx->LevelIndex, it,
static_cast<bool>(FreshSegment), firstLsn, lastLsn, restoreDeadline, partitionKey)
, CompactionID(TAppData::RandomProvider->GenRand64())
, SkeletonId(rtCtx->SkeletonId)
diff --git a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h
index 57a478a1a2..6fa069c703 100644
--- a/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h
+++ b/ydb/core/blobstorage/vdisk/hullop/blobstorage_hullcompactworker.h
@@ -116,7 +116,7 @@ namespace NKikimr {
THullCtxPtr HullCtx;
TPDiskCtxPtr PDiskCtx;
THugeBlobCtxPtr HugeBlobCtx;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
// Group Type
const TBlobStorageGroupType GType;
@@ -290,7 +290,7 @@ namespace NKikimr {
THullCompactionWorker(THullCtxPtr hullCtx,
TPDiskCtxPtr pdiskCtx,
THugeBlobCtxPtr hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TIntrusivePtr<TLevelIndex> levelIndex,
const TIterator& it,
bool isFresh,
@@ -301,7 +301,7 @@ namespace NKikimr {
: HullCtx(std::move(hullCtx))
, PDiskCtx(std::move(pdiskCtx))
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
, GType(HullCtx->VCtx->Top->GType)
, LevelIndex(std::move(levelIndex))
, FirstLsn(firstLsn)
@@ -570,7 +570,7 @@ namespace NKikimr {
bool PreprocessItem() {
// finish merging data for this item
if constexpr (LogoBlobs) {
- IndexMerger.Finish(HugeBlobCtx->IsHugeBlob(GType, Key.LogoBlobID(), MinHugeBlobInBytes));
+ IndexMerger.Finish(HugeBlobCtx->IsHugeBlob(GType, Key.LogoBlobID(), MinREALHugeBlobInBytes));
} else {
IndexMerger.Finish(false);
}
diff --git a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
index c71d6c3524..cda349b521 100644
--- a/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
+++ b/ydb/core/blobstorage/vdisk/localrecovery/localrecovery_public.cpp
@@ -444,6 +444,7 @@ namespace NKikimr {
LocRecCtx->PDiskCtx->Dsk->ChunkSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
+ Config->OldMinHugeBlobInBytes,
Config->MilestoneHugeBlobInBytes,
Config->MaxLogoBlobDataSize,
Config->HugeBlobOverhead,
@@ -465,6 +466,7 @@ namespace NKikimr {
LocRecCtx->PDiskCtx->Dsk->ChunkSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
LocRecCtx->PDiskCtx->Dsk->AppendBlockSize,
+ Config->OldMinHugeBlobInBytes,
Config->MilestoneHugeBlobInBytes,
Config->MaxLogoBlobDataSize,
Config->HugeBlobOverhead,
diff --git a/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp b/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp
index daf51e2ec3..7a5522c8c9 100644
--- a/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp
+++ b/ydb/core/blobstorage/vdisk/query/query_spacetracker_ut.cpp
@@ -32,7 +32,7 @@ namespace NKikimr {
costSettings.SetWriteSpeedBps(Max<ui64>());
costSettings.SetReadBlockSize(Max<ui64>());
costSettings.SetWriteBlockSize(Max<ui64>());
- costSettings.SetMinHugeBlobInBytes(Max<ui32>());
+ costSettings.SetMinREALHugeBlobInBytes(Max<ui32>());
return costSettings;
}
diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp b/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp
index a9aef05686..41bc8c156d 100644
--- a/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp
+++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_repl.cpp
@@ -156,7 +156,7 @@ namespace NKikimr {
};
std::shared_ptr<TReplCtx> ReplCtx;
- ui32 NextMinHugeBlobInBytes;
+ ui32 NextMinREALHugeBlobInBytes;
THistory History;
EState State;
TInstant LastReplStart;
@@ -257,7 +257,7 @@ namespace NKikimr {
}
void Handle(TEvMinHugeBlobSizeUpdate::TPtr ev) {
- NextMinHugeBlobInBytes = ev->Get()->MinHugeBlobInBytes;
+ NextMinREALHugeBlobInBytes = ev->Get()->MinREALHugeBlobInBytes;
}
void StartReplication() {
@@ -270,8 +270,8 @@ namespace NKikimr {
ReplCtx->MonGroup.ReplWorkUnitsDone() = 0;
ReplCtx->MonGroup.ReplItemsRemaining() = 0;
ReplCtx->MonGroup.ReplItemsDone() = 0;
- Y_ABORT_UNLESS(NextMinHugeBlobInBytes);
- ReplCtx->MinHugeBlobInBytes = NextMinHugeBlobInBytes;
+ Y_ABORT_UNLESS(NextMinREALHugeBlobInBytes);
+ ReplCtx->MinREALHugeBlobInBytes = NextMinREALHugeBlobInBytes;
UnrecoveredNonphantomBlobs = false;
Become(&TThis::StateRepl);
@@ -697,7 +697,7 @@ namespace NKikimr {
TReplScheduler(std::shared_ptr<TReplCtx> &replCtx)
: TActorBootstrapped<TReplScheduler>()
, ReplCtx(replCtx)
- , NextMinHugeBlobInBytes(ReplCtx->MinHugeBlobInBytes)
+ , NextMinREALHugeBlobInBytes(ReplCtx->MinREALHugeBlobInBytes)
, History(HistorySize)
, State(Relaxation)
, ReplProgressWatchdog(
diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h b/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h
index 2d2211f63c..a8ac714d52 100644
--- a/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h
+++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_replctx.h
@@ -15,7 +15,7 @@ namespace NKikimr {
TIntrusivePtr<THullCtx> HullCtx;
TPDiskCtxPtr PDiskCtx;
std::shared_ptr<THugeBlobCtx> HugeBlobCtx;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
TIntrusivePtr<THullDs> HullDs;
TIntrusivePtr<TBlobStorageGroupInfo> GInfo;
TActorId SkeletonId;
@@ -31,7 +31,7 @@ namespace NKikimr {
TIntrusivePtr<THullCtx> hullCtx,
TPDiskCtxPtr pdiskCtx,
std::shared_ptr<THugeBlobCtx> hugeBlobCtx,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TIntrusivePtr<THullDs> hullDs,
TIntrusivePtr<TBlobStorageGroupInfo> info,
const TActorId &skeletonId,
@@ -42,7 +42,7 @@ namespace NKikimr {
, HullCtx(std::move(hullCtx))
, PDiskCtx(std::move(pdiskCtx))
, HugeBlobCtx(std::move(hugeBlobCtx))
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
, HullDs(std::move(hullDs))
, GInfo(std::move(info))
, SkeletonId(skeletonId)
@@ -51,7 +51,7 @@ namespace NKikimr {
, PDiskWriteBytes(std::move(pdiskWriteBytes))
, PausedAtStart(pausedAtStart)
{
- Y_ABORT_UNLESS(MinHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
}
bool GetAddHeader() const { return !HullCtx || HullCtx->AddHeader; }
diff --git a/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h b/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h
index 3de2562e82..10dcccd3dc 100644
--- a/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h
+++ b/ydb/core/blobstorage/vdisk/repl/blobstorage_replrecoverymachine.h
@@ -206,7 +206,7 @@ namespace NKikimr {
partsSize += partSize;
TRope& data = item.Parts[i];
Y_ABORT_UNLESS(data.GetSize() == partSize);
- if (ReplCtx->HugeBlobCtx->IsHugeBlob(groupType, id, ReplCtx->MinHugeBlobInBytes)) {
+ if (ReplCtx->HugeBlobCtx->IsHugeBlob(groupType, id, ReplCtx->MinREALHugeBlobInBytes)) {
AddBlobToQueue(partId, TDiskBlob::Create(id.BlobSize(), i + 1, groupType.TotalPartCount(),
std::move(data), Arena, ReplCtx->GetAddHeader()), {}, true, rbq);
++numHuge;
@@ -380,7 +380,7 @@ namespace NKikimr {
void RecoverMetadata(const TLogoBlobID& id, TRecoveredBlobsQueue& rbq) {
while (!MetadataParts.empty() && MetadataParts.front().FullID() <= id) {
const TLogoBlobID id = MetadataParts.front();
- const bool isHugeBlob = ReplCtx->HugeBlobCtx->IsHugeBlob(ReplCtx->VCtx->Top->GType, id.FullID(), ReplCtx->MinHugeBlobInBytes);
+ const bool isHugeBlob = ReplCtx->HugeBlobCtx->IsHugeBlob(ReplCtx->VCtx->Top->GType, id.FullID(), ReplCtx->MinREALHugeBlobInBytes);
MetadataParts.pop_front();
STLOG(PRI_DEBUG, BS_REPL, BSVR30, VDISKP(ReplCtx->VCtx->VDiskLogPrefix,
"TRecoveryMachine::RecoverMetadata"), (BlobId, id));
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
index def1404392..0832fbc623 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeleton.cpp
@@ -202,25 +202,25 @@ namespace NKikimr {
if (!ApplyHugeBlobSize(type.GetMinHugeBlobSizeInBytes())) {
continue;
}
- Y_ABORT_UNLESS(MinHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
if (Config->RunRepl) {
- ctx.Send(Db->ReplID, new TEvMinHugeBlobSizeUpdate(MinHugeBlobInBytes));
+ ctx.Send(Db->ReplID, new TEvMinHugeBlobSizeUpdate(MinREALHugeBlobInBytes));
}
if (Hull) {
- Hull->ApplyHugeBlobSize(MinHugeBlobInBytes, ctx);
+ Hull->ApplyHugeBlobSize(MinREALHugeBlobInBytes, ctx);
}
- ctx.Send(*SkeletonFrontIDPtr, new TEvMinHugeBlobSizeUpdate(MinHugeBlobInBytes));
+ ctx.Send(*SkeletonFrontIDPtr, new TEvMinHugeBlobSizeUpdate(MinREALHugeBlobInBytes));
}
}
}
bool ApplyHugeBlobSize(ui32 minHugeBlobInBytes) {
- const ui32 alignedSize = HugeBlobCtx->HugeSlotsMap->AlignByBlockSize(minHugeBlobInBytes);
- if (MinHugeBlobInBytes == alignedSize) {
+ ui32 alignedSize = HugeBlobCtx->HugeSlotsMap->AlignByBlockSize(minHugeBlobInBytes) + 1;
+ if (MinREALHugeBlobInBytes == alignedSize) {
return false;
}
- MinHugeBlobInBytes = alignedSize;
- IFaceMonGroup->MinHugeBlobInBytes(MinHugeBlobInBytes);
+ MinREALHugeBlobInBytes = alignedSize;
+ IFaceMonGroup->MinHugeBlobInBytes(MinREALHugeBlobInBytes);
return true;
}
@@ -583,7 +583,7 @@ namespace NKikimr {
TVPutInfo &info = putsInfo.back();
try {
- info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, blobId.FullID(), MinHugeBlobInBytes);
+ info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, blobId.FullID(), MinREALHugeBlobInBytes);
if (info.IsHugeBlob) {
LOG_CRIT_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << "TEvVMultiPut: TEvVMultiPut has huge blob# "
<< blobId << " Marker# BSVS08");
@@ -726,7 +726,7 @@ namespace NKikimr {
const ui64 bufSize = info.Buffer.GetSize();
try {
- info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, id.FullID(), MinHugeBlobInBytes);
+ info.IsHugeBlob = HugeBlobCtx->IsHugeBlob(VCtx->Top->GType, id.FullID(), MinREALHugeBlobInBytes);
} catch (yexception ex) {
LOG_ERROR_S(ctx, BS_VDISK_PUT, VCtx->VDiskLogPrefix << ex.what() << " Marker# BSVS41");
info.HullStatus = {NKikimrProto::ERROR, "", false};
@@ -1788,7 +1788,7 @@ namespace NKikimr {
auto msg = std::make_unique<TEvFrontRecoveryStatus>(TEvFrontRecoveryStatus::SyncGuidRecoveryDone,
NKikimrProto::OK,
(PDiskCtx ? PDiskCtx->Dsk : nullptr),
- MinHugeBlobInBytes,
+ MinREALHugeBlobInBytes,
Db->GetVDiskIncarnationGuid());
ctx.Send(*SkeletonFrontIDPtr, msg.release());
ctx.Send(NConsole::MakeConfigsDispatcherID(SelfId().NodeId()), new NConsole::TEvConfigsDispatcher::TEvSetConfigSubscriptionRequest(
@@ -1815,7 +1815,7 @@ namespace NKikimr {
auto msg = std::make_unique<TEvFrontRecoveryStatus>(phase,
NKikimrProto::ERROR,
(PDiskCtx ? PDiskCtx->Dsk : nullptr),
- MinHugeBlobInBytes,
+ MinREALHugeBlobInBytes,
Db->GetVDiskIncarnationGuid());
ctx.Send(*SkeletonFrontIDPtr, msg.release());
// push the status
@@ -1863,7 +1863,7 @@ namespace NKikimr {
// check status
if (ev->Get()->Status == NKikimrProto::OK) {
ApplyHugeBlobSize(Config->MinHugeBlobInBytes);
- Y_ABORT_UNLESS(MinHugeBlobInBytes);
+ Y_ABORT_UNLESS(MinREALHugeBlobInBytes);
// handle special case when donor disk starts and finds out that it has been wiped out
if (ev->Get()->LsnMngr->GetOriginallyRecoveredLsn() == 0 && Config->BaseInfo.DonorMode) {
@@ -1881,7 +1881,7 @@ namespace NKikimr {
auto msg = std::make_unique<TEvFrontRecoveryStatus>(TEvFrontRecoveryStatus::LocalRecoveryDone,
NKikimrProto::OK,
PDiskCtx->Dsk,
- MinHugeBlobInBytes,
+ MinREALHugeBlobInBytes,
Db->GetVDiskIncarnationGuid());
ctx.Send(*SkeletonFrontIDPtr, msg.release());
@@ -1943,7 +1943,7 @@ namespace NKikimr {
Db->HugeKeeperID);
// create Hull
- Hull = std::make_shared<THull>(Db->LsnMngr, PDiskCtx, HugeBlobCtx, MinHugeBlobInBytes,
+ Hull = std::make_shared<THull>(Db->LsnMngr, PDiskCtx, HugeBlobCtx, MinREALHugeBlobInBytes,
Db->SkeletonID, Config->BalancingEnableDelete, std::move(*ev->Get()->Uncond),
ctx.ExecutorThread.ActorSystem, Config->BarrierValidation, Db->HugeKeeperID);
ActiveActors.Insert(Hull->RunHullServices(Config, HullLogCtx, Db->SyncLogFirstLsnToKeep,
@@ -2044,7 +2044,7 @@ namespace NKikimr {
DbBirthLsn = ev->Get()->DbBirthLsn;
SkeletonIsUpAndRunning(ctx, Config->RunRepl);
if (Config->RunRepl) {
- auto replCtx = std::make_shared<TReplCtx>(VCtx, HullCtx, PDiskCtx, HugeBlobCtx, MinHugeBlobInBytes, Hull->GetHullDs(),
+ auto replCtx = std::make_shared<TReplCtx>(VCtx, HullCtx, PDiskCtx, HugeBlobCtx, MinREALHugeBlobInBytes, Hull->GetHullDs(),
GInfo, SelfId(), Config, PDiskWriteBytes, Config->ReplPausedAtStart);
Db->ReplID.Set(ctx.Register(CreateReplActor(replCtx)));
ActiveActors.Insert(Db->ReplID, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE); // keep forever
@@ -2594,7 +2594,7 @@ namespace NKikimr {
.TimeToSleepIfNothingToDo=Config->BalancingTimeToSleepIfNothingToDo,
};
auto balancingCtx = std::make_shared<TBalancingCtx>(
- balancingCfg, VCtx, PDiskCtx, HugeBlobCtx, SelfId(), Hull->GetSnapshot(), Config, GInfo, MinHugeBlobInBytes);
+ balancingCfg, VCtx, PDiskCtx, HugeBlobCtx, SelfId(), Hull->GetSnapshot(), Config, GInfo, MinREALHugeBlobInBytes);
BalancingId = RunInBatchPool(ctx, CreateBalancingActor(balancingCtx));
ActiveActors.Insert(BalancingId, __FILE__, __LINE__, ctx, NKikimrServices::BLOBSTORAGE);
}
@@ -2857,7 +2857,7 @@ namespace NKikimr {
THullCtxPtr HullCtx;
THugeBlobCtxPtr HugeBlobCtx;
std::shared_ptr<THullLogCtx> HullLogCtx;
- ui32 MinHugeBlobInBytes = 0;
+ ui32 MinREALHugeBlobInBytes = 0;
std::shared_ptr<THull> Hull; // run it after local recovery
std::shared_ptr<TOutOfSpaceLogic> OutOfSpaceLogic;
std::shared_ptr<TQueryCtx> QueryCtx;
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
index 7d9f49f6cd..9d9b820c54 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.cpp
@@ -42,12 +42,12 @@ namespace NKikimr {
TEvFrontRecoveryStatus::TEvFrontRecoveryStatus(EPhase phase,
NKikimrProto::EReplyStatus status,
const TIntrusivePtr<TPDiskParams> &dsk,
- ui32 minHugeBlobInBytes,
+ ui32 minREALHugeBlobInBytes,
TVDiskIncarnationGuid vdiskIncarnationGuid)
: Phase(phase)
, Status(status)
, Dsk(dsk)
- , MinHugeBlobInBytes(minHugeBlobInBytes)
+ , MinREALHugeBlobInBytes(minREALHugeBlobInBytes)
, VDiskIncarnationGuid(vdiskIncarnationGuid)
{}
@@ -816,7 +816,7 @@ namespace NKikimr {
TBlobStorageGroupType type = (GInfo ? GInfo->Type : TErasureType::ErasureNone);
VCtx->UpdateCostModel(std::make_unique<TCostModel>(msg->Dsk->SeekTimeUs, msg->Dsk->ReadSpeedBps,
msg->Dsk->WriteSpeedBps, msg->Dsk->ReadBlockSize, msg->Dsk->WriteBlockSize,
- msg->MinHugeBlobInBytes, type));
+ msg->MinREALHugeBlobInBytes, type));
break;
}
case TEvFrontRecoveryStatus::SyncGuidRecoveryDone:
@@ -831,7 +831,7 @@ namespace NKikimr {
void Handle(TEvMinHugeBlobSizeUpdate::TPtr &ev) {
VCtx->UpdateCostModel(std::make_unique<TCostModel>(VCtx->CostModel->SeekTimeUs, VCtx->CostModel->ReadSpeedBps,
VCtx->CostModel->WriteSpeedBps, VCtx->CostModel->ReadBlockSize, VCtx->CostModel->WriteBlockSize,
- ev->Get()->MinHugeBlobInBytes, VCtx->CostModel->GType));
+ ev->Get()->MinREALHugeBlobInBytes, VCtx->CostModel->GType));
}
static NKikimrWhiteboard::EFlag ToLightSignal(NKikimrWhiteboard::EVDiskState st) {
diff --git a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h
index 068edefb02..d5a7c13ad6 100644
--- a/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h
+++ b/ydb/core/blobstorage/vdisk/skeleton/blobstorage_skeletonfront.h
@@ -24,13 +24,13 @@ namespace NKikimr {
const EPhase Phase;
const NKikimrProto::EReplyStatus Status;
const TIntrusivePtr<TPDiskParams> Dsk;
- ui32 MinHugeBlobInBytes;
+ ui32 MinREALHugeBlobInBytes;
const TVDiskIncarnationGuid VDiskIncarnationGuid;
TEvFrontRecoveryStatus(EPhase phase,
NKikimrProto::EReplyStatus status,
const TIntrusivePtr<TPDiskParams> &dsk,
- ui32 MinHugeBlobInBytes,
+ ui32 MinREALHugeBlobInBytes,
TVDiskIncarnationGuid vdiskIncarnationGuid);
~TEvFrontRecoveryStatus();
};
diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto
index 554ca1febb..3c4aa292d2 100644
--- a/ydb/core/protos/blobstorage.proto
+++ b/ydb/core/protos/blobstorage.proto
@@ -25,7 +25,7 @@ message TVDiskCostSettings {
optional uint64 WriteSpeedBps = 3;
optional uint64 ReadBlockSize = 4;
optional uint64 WriteBlockSize = 5;
- optional uint32 MinHugeBlobInBytes = 6;
+ optional uint32 MinREALHugeBlobInBytes = 6;
};
message TWindowFeedback {