summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKirill Pleshivtsev <[email protected]>2026-07-08 18:39:37 +0700
committerGitHub <[email protected]>2026-07-08 11:39:37 +0000
commit779aa20541be76d26bffe160acf8527093a790fb (patch)
tree9f78f5a39d930e92365b65b52758b1ce39c46907
parent60af8e5376cfd271de20ff519123243a40ca9d9d (diff)
[NBS] Cool down flush requests when node disconnected (#45894)
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.cpp20
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.h7
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp5
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp6
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h8
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map.cpp20
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map_ut.cpp2
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp16
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.h4
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp92
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.cpp54
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.h2
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request_ut.cpp55
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.cpp9
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.h1
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask_ut.cpp17
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.cpp5
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.h3
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat_ut.cpp31
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp34
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h15
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle_ut.cpp115
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.cpp10
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.h4
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.cpp91
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.h3
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/region.cpp11
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp5
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.cpp39
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.h1
-rw-r--r--ydb/core/nbs/cloud/blockstore/libs/storage/storage_transport/ic_storage_transport_actor.h1
31 files changed, 514 insertions, 172 deletions
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.cpp
index b12d55d125f..aff8a007246 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.cpp
@@ -75,9 +75,8 @@ TString ToString(const TLogTitle::TVolume& data)
{
TStringBuilder stream;
- stream << "[v:" << data.TabletId;
- stream << " g:" << TOptional{data.Generation};
- stream << " d:" << TOptional{data.DiskId};
+ stream << "[v:" << TOptional{data.DiskId} << "/" << data.TabletId << "/"
+ << data.Generation;
return stream;
}
@@ -86,9 +85,8 @@ TString ToString(const TLogTitle::TPartitionDirect& data)
{
TStringBuilder stream;
- stream << "[pd:" << data.TabletId;
- stream << " g:" << TOptional{data.Generation};
- stream << " d:" << TOptional{data.DiskId};
+ stream << "[pd:" << TOptional{data.DiskId} << "/" << data.TabletId << "/"
+ << data.Generation;
return stream;
}
@@ -97,10 +95,8 @@ TString ToString(const TLogTitle::TDirectBlockGroup& data)
{
TStringBuilder stream;
- stream << "[dbg:" << data.TabletId;
- stream << " g:" << TOptional{data.Generation};
- stream << " d:" << TOptional{data.DiskId};
- stream << " indx:" << data.DirectBlockGroupIndex;
+ stream << "[dbg:" << data.DiskId << "/" << data.DBGIndex;
+ stream << " t:" << TOptional{data.TabletId} << "/" << data.Generation;
return stream;
}
@@ -109,8 +105,8 @@ TString ToString(const TLogTitle::TVChunk& data)
{
TStringBuilder stream;
- stream << "[vchk:" << data.DiskId;
- stream << " indx:" << data.VChunkIndex;
+ stream << "[vchk:" << data.DiskId << "/" << data.DBGIndex << "/"
+ << data.VChunkIndex;
return stream;
}
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.h b/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.h
index b7021add347..14f10657f41 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/model/log_title.h
@@ -22,7 +22,7 @@ public:
struct TVolume
{
- ui64 TabletId;
+ ui64 TabletId = 0;
TString DiskId;
ui32 Generation = 0;
};
@@ -37,15 +37,16 @@ public:
struct TDirectBlockGroup
{
TString DiskId;
+ size_t DBGIndex = 0;
ui64 TabletId = 0;
ui32 Generation = 0;
- size_t DirectBlockGroupIndex = 0;
};
struct TVChunk
{
TString DiskId;
- ui32 VChunkIndex;
+ ui32 DBGIndex = 0;
+ ui32 VChunkIndex = 0;
};
struct TDDiskDataCopier
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp
index 68fd981751e..66fea1c07c4 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_impl.cpp
@@ -109,10 +109,9 @@ TDirectBlockGroup::TDirectBlockGroup(
GetCycleCount(),
TLogTitle::TDirectBlockGroup{
.DiskId = diskId,
+ .DBGIndex = DirectBlockGroupIndex,
.TabletId = TabletId,
- .Generation = TabletGeneration,
- .DirectBlockGroupIndex = DirectBlockGroupIndex,
- })
+ .Generation = TabletGeneration})
, Oracle(StorageConfig, this)
{
Y_ASSERT(pbufferIds.size() == DirectBlockGroupHostCount);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp
index bd9fc26b7ef..44f7fa95dba 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.cpp
@@ -98,6 +98,12 @@ TDuration TOracleMock::GetIndirectWriteReplyTimeout() const
return PBufferReplyTimeout;
}
+TDuration TOracleMock::GetFlushRequestCooldown(THostMask hosts) const
+{
+ Y_UNUSED(hosts);
+ return FlushRequestCooldown;
+}
+
TDuration TOracleMock::GetFlushRequestTimeout() const
{
return FlushRequestTimeout;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h
index 2028d9cde91..ad233132a2a 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/direct_block_group_mock.h
@@ -16,6 +16,7 @@ struct TOracleMock: public IOracle
TDuration WriteRequestTimeout;
TDuration PBufferReplyTimeout;
EWriteMode WriteMode = EWriteMode::DirectWrite;
+ TDuration FlushRequestCooldown;
TDuration FlushRequestTimeout;
TDuration EraseRequestTimeout;
TVector<THostStat> HostStatistics;
@@ -50,14 +51,19 @@ struct TOracleMock: public IOracle
THostIndex host,
EDataLocation dataLocation) const override;
[[nodiscard]] TDuration GetReadRequestTimeout() const override;
+
+ [[nodiscard]] EWriteMode GetWriteMode() const override;
[[nodiscard]] TDuration GetWriteHedgingDelay(
THostMask hosts,
bool indirect) const override;
[[nodiscard]] TDuration GetWriteRequestTimeout() const override;
[[nodiscard]] TDuration GetIndirectWriteReplyTimeout() const override;
+
+ [[nodiscard]] TDuration GetFlushRequestCooldown(
+ THostMask hosts) const override;
[[nodiscard]] TDuration GetFlushRequestTimeout() const override;
+
[[nodiscard]] TDuration GetEraseRequestTimeout() const override;
- [[nodiscard]] EWriteMode GetWriteMode() const override;
[[nodiscard]] const THostStat& GetHostStatistics(
THostIndex hostIndex) const override;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map.cpp
index be8c308cf9a..97b67d25d44 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map.cpp
@@ -338,6 +338,8 @@ void TBlocksDirtyMap::UpdateConfig(const TVChunkConfig& vChunkConfig)
for (auto lsn: erased) {
Inflight.RemoveRange(lsn);
+ ReadyToErase.erase(lsn);
+ ReadyToFlush.erase(lsn);
}
}
@@ -572,12 +574,16 @@ void TBlocksDirtyMap::FlushFinished(
const TVector<ui64>& flushOk,
const TVector<ui64>& flushFailed)
{
+ if (DisabledHosts.Get(route.DestinationHostIndex)) {
+ return;
+ }
+
for (ui64 lsn: flushOk) {
auto item = Inflight.GetValue(lsn);
Y_ABORT_UNLESS(item);
auto& inflight = item->Value;
- inflight.ConfirmFlush(route);
+ inflight.ConfirmFlush(route.DestinationHostIndex);
}
for (ui64 lsn: flushFailed) {
@@ -585,7 +591,7 @@ void TBlocksDirtyMap::FlushFinished(
Y_ABORT_UNLESS(item);
auto& inflight = item->Value;
- inflight.FlushFailed(route);
+ inflight.FlushFailed(route.DestinationHostIndex);
}
}
@@ -596,7 +602,10 @@ void TBlocksDirtyMap::EraseFinished(
{
for (ui64 lsn: eraseOk) {
auto item = Inflight.GetValue(lsn);
- Y_ABORT_UNLESS(item);
+ if (!item) {
+ // The item was deleted when the host was disabled.
+ continue;
+ }
auto& inflight = item->Value;
if (inflight.ConfirmErase(host)) {
@@ -607,7 +616,10 @@ void TBlocksDirtyMap::EraseFinished(
for (ui64 lsn: eraseFailed) {
auto item = Inflight.GetValue(lsn);
- Y_ABORT_UNLESS(item);
+ if (!item) {
+ // The item was deleted when the host was disabled.
+ continue;
+ }
auto& inflight = item->Value;
inflight.EraseFailed(host);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map_ut.cpp
index e9afbf2f95f..90da23a359b 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map_ut.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/dirty_map_ut.cpp
@@ -17,7 +17,7 @@ constexpr ui64 DefaultVChunkSize = RegionSize / DirectBlockGroupsCount;
TVChunkConfig MakeTestVChunkConfig()
{
return TVChunkConfig::MakeDefault(
- /*vChunkIndex=*/0,
+ 0, // VChunkIndex
DirectBlockGroupHostCount,
DefaultPrimaryCount);
}
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp
index 50c5ad73bea..2e0b2cc1c14 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.cpp
@@ -191,13 +191,13 @@ THostIndex TInflightInfo::RequestFlush(
Y_ABORT_UNLESS(false);
}
-void TInflightInfo::ConfirmFlush(THostRoute route)
+void TInflightInfo::ConfirmFlush(THostIndex host)
{
Y_ABORT_UNLESS(State == EState::PBufferFlushing);
- Y_ABORT_UNLESS(FlushRequested.Get(route.DestinationHostIndex));
- Y_ABORT_UNLESS(!FlushConfirmed.Get(route.DestinationHostIndex));
+ Y_ABORT_UNLESS(FlushRequested.Get(host));
+ Y_ABORT_UNLESS(!FlushConfirmed.Get(host));
- FlushConfirmed.Set(route.DestinationHostIndex);
+ FlushConfirmed.Set(host);
if (FlushDesired == FlushConfirmed) {
SetState(EState::PBufferFlushed);
@@ -208,13 +208,13 @@ void TInflightInfo::ConfirmFlush(THostRoute route)
}
}
-void TInflightInfo::FlushFailed(THostRoute route)
+void TInflightInfo::FlushFailed(THostIndex host)
{
Y_ABORT_UNLESS(State == EState::PBufferFlushing);
- Y_ABORT_UNLESS(FlushRequested.Get(route.DestinationHostIndex));
- Y_ABORT_UNLESS(!FlushConfirmed.Get(route.DestinationHostIndex));
+ Y_ABORT_UNLESS(FlushRequested.Get(host));
+ Y_ABORT_UNLESS(!FlushConfirmed.Get(host));
- FlushRequested.Reset(route.DestinationHostIndex);
+ FlushRequested.Reset(host);
ReadyQueue->Register(Lsn, IReadyQueue::EQueueType::Flush);
}
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.h
index e90b8feeef8..28b2895469c 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info.h
@@ -148,8 +148,8 @@ public:
[[nodiscard]] THostIndex RequestFlush(
THostIndex destination,
THostMask disabledHosts);
- void ConfirmFlush(THostRoute route);
- void FlushFailed(THostRoute route);
+ void ConfirmFlush(THostIndex host);
+ void FlushFailed(THostIndex host);
[[nodiscard]] THostMask GetRequestedFlushes() const;
void RequestErase(THostIndex host);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp
index 113b79e36db..cf46e88e604 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/dirty_map/inflight_info_ut.cpp
@@ -120,17 +120,11 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
inflightInfo.RequestFlush(THostIndex{2}, THostMask()));
// Confirm flushes
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
UNIT_ASSERT_VALUES_EQUAL(false, readyQueue.ReadyToErase.contains(123));
- inflightInfo.ConfirmFlush(
- {.SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
+ inflightInfo.ConfirmFlush(THostIndex{1});
UNIT_ASSERT_VALUES_EQUAL(false, readyQueue.ReadyToErase.contains(123));
- inflightInfo.ConfirmFlush(
- {.SourceHostIndex = THostIndex{2},
- .DestinationHostIndex = THostIndex{2}});
+ inflightInfo.ConfirmFlush(THostIndex{2});
UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToErase.contains(123));
// Check erase requests
@@ -164,21 +158,15 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
Y_UNUSED(l);
// Confirm two flushes
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
- inflightInfo.ConfirmFlush(
- {.SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
+ inflightInfo.ConfirmFlush(THostIndex{1});
// Check lock/unlock PBuffer
inflightInfo.LockPBuffer();
UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToErase.empty());
// Confirm last flush
- inflightInfo.ConfirmFlush(
- {.SourceHostIndex = THostIndex{2},
- .DestinationHostIndex = THostIndex{2}});
+ inflightInfo.ConfirmFlush(THostIndex{2});
UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToErase.empty());
inflightInfo.UnlockPBuffer();
@@ -220,9 +208,7 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
// When a flush fails, the lsn must be queued for a flush again.
readyQueue.ReadyToFlush.clear();
- inflightInfo.FlushFailed(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
+ inflightInfo.FlushFailed(THostIndex{0});
UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToFlush.contains(123));
// Restart flush to host 0
@@ -231,17 +217,11 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
inflightInfo.RequestFlush(THostIndex{0}, THostMask()));
// Confirm flushes
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
UNIT_ASSERT_VALUES_EQUAL(false, readyQueue.ReadyToErase.contains(123));
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
+ inflightInfo.ConfirmFlush(THostIndex{1});
UNIT_ASSERT_VALUES_EQUAL(false, readyQueue.ReadyToErase.contains(123));
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{2},
- .DestinationHostIndex = THostIndex{2}});
+ inflightInfo.ConfirmFlush(THostIndex{2});
UNIT_ASSERT_VALUES_EQUAL(true, readyQueue.ReadyToErase.contains(123));
// Erase started
@@ -318,15 +298,9 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
2,
inflightInfo.RequestFlush(THostIndex{2}, THostMask()));
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{2},
- .DestinationHostIndex = THostIndex{2}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
+ inflightInfo.ConfirmFlush(THostIndex{1});
+ inflightInfo.ConfirmFlush(THostIndex{2});
// Before any erase, all written hosts need erasing.
auto eraseNeeded = inflightInfo.GetEraseNeeded();
@@ -385,15 +359,9 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
2,
inflightInfo.RequestFlush(THostIndex{2}, THostMask()));
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{2},
- .DestinationHostIndex = THostIndex{2}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
+ inflightInfo.ConfirmFlush(THostIndex{1});
+ inflightInfo.ConfirmFlush(THostIndex{2});
// Request erase for host 0 and then fail it.
inflightInfo.RequestErase(THostIndex{0});
@@ -563,12 +531,8 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
Y_UNUSED(l);
// Confirm flush for hosts 0 and 1 only.
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
+ inflightInfo.ConfirmFlush(THostIndex{1});
UNIT_ASSERT_VALUES_EQUAL(
TInflightInfo::EState::PBufferFlushing,
@@ -600,15 +564,9 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
l = inflightInfo.RequestFlush(THostIndex{1}, THostMask());
l = inflightInfo.RequestFlush(THostIndex{2}, THostMask());
Y_UNUSED(l);
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{2},
- .DestinationHostIndex = THostIndex{2}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
+ inflightInfo.ConfirmFlush(THostIndex{1});
+ inflightInfo.ConfirmFlush(THostIndex{2});
// Start erasing hosts 0 and 1, confirm both.
inflightInfo.RequestErase(THostIndex{0});
@@ -688,12 +646,8 @@ Y_UNIT_TEST_SUITE(TInflightInfoTests)
l = inflightInfo.RequestFlush(THostIndex{1}, THostMask());
l = inflightInfo.RequestFlush(THostIndex{2}, THostMask());
Y_UNUSED(l);
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{0},
- .DestinationHostIndex = THostIndex{0}});
- inflightInfo.ConfirmFlush(THostRoute{
- .SourceHostIndex = THostIndex{1},
- .DestinationHostIndex = THostIndex{1}});
+ inflightInfo.ConfirmFlush(THostIndex{0});
+ inflightInfo.ConfirmFlush(THostIndex{1});
UNIT_ASSERT_VALUES_EQUAL(
TInflightInfo::EState::PBufferFlushing,
inflightInfo.GetState());
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.cpp
index 69fa7dd04e0..da267134c90 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.cpp
@@ -31,6 +31,8 @@ TFlushRequestExecutor::TFlushRequestExecutor(
, Span(std::move(span))
, Route(route)
, Hint(std::move(hint))
+ , Cooldown(DirectBlockGroup->GetOracle()->GetFlushRequestCooldown(
+ THostMask::MakeFromRoute(Route)))
, RequestTimeout(DirectBlockGroup->GetOracle()->GetFlushRequestTimeout())
{
Y_ABORT_UNLESS(Route.SourceHostIndex != InvalidHostIndex);
@@ -52,21 +54,22 @@ TFlushRequestExecutor::~TFlushRequestExecutor()
void TFlushRequestExecutor::Run()
{
- ScheduleRequestTimeout();
-
- auto future = DirectBlockGroup->SyncWithPBuffer(
- VChunkConfig.GetVChunkIndex(),
- Route.SourceHostIndex,
- Route.DestinationHostIndex,
- Hint.Segments,
- Span.GetTraceId());
- future.Subscribe(
- [self = shared_from_this()] //
- (const NThreading::TFuture<TDBGFlushResponse>& f)
- {
- //
- self->OnFlushResponse(f.GetValue());
- });
+ if (!Cooldown) {
+ DoRun();
+ } else {
+ LOG_INFO(
+ *ActorSystem,
+ NKikimrServices::NBS_PARTITION,
+ "%s Flush cooldown time %s",
+ LogTitle.GetWithTime().c_str(),
+ FormatDuration(Cooldown).c_str());
+ DirectBlockGroup->Schedule(
+ Cooldown,
+ [self = shared_from_this()]()
+ {
+ self->DoRun(); //
+ });
+ }
}
TString TFlushRequestExecutor::Print()
@@ -74,6 +77,9 @@ TString TFlushRequestExecutor::Print()
TStringBuilder result;
result << LogTitle.GetWithTime();
result << Hint.DebugPrint(true);
+ if (Cooldown) {
+ result << ",Cooldown=" << FormatDuration(Cooldown);
+ }
result << (Promise.IsReady() ? ",Replied" : ",NotReplied");
return result;
}
@@ -84,6 +90,24 @@ TFlushRequestExecutor::GetFuture() const
return Promise.GetFuture();
}
+void TFlushRequestExecutor::DoRun()
+{
+ ScheduleRequestTimeout();
+
+ auto future = DirectBlockGroup->SyncWithPBuffer(
+ VChunkConfig.GetVChunkIndex(),
+ Route.SourceHostIndex,
+ Route.DestinationHostIndex,
+ Hint.Segments,
+ Span.GetTraceId());
+ future.Subscribe(
+ [self = shared_from_this()] //
+ (const NThreading::TFuture<TDBGFlushResponse>& f)
+ {
+ self->OnFlushResponse(f.GetValue()); //
+ });
+}
+
void TFlushRequestExecutor::OnFlushResponse(const TDBGFlushResponse& response)
{
Y_ABORT_UNLESS(Hint.Segments.size() == response.Errors.size());
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.h
index 62b40a1b702..9f007e59f3d 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request.h
@@ -43,6 +43,7 @@ public:
NThreading::TFuture<TResponse> GetFuture() const;
private:
+ void DoRun();
void SendFlushRequest(THostIndex host);
void OnFlushResponse(const TDBGFlushResponse& response);
void Reply(TVector<ui64> flushOk, TVector<ui64> flushFailed);
@@ -57,6 +58,7 @@ private:
const NWilson::TSpan Span;
const THostRoute Route;
const TFlushHint Hint;
+ const TDuration Cooldown;
const TDuration RequestTimeout;
NThreading::TPromise<TResponse> Promise =
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request_ut.cpp
index 53580240731..8a1855a9ecb 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request_ut.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/flush_request_ut.cpp
@@ -180,6 +180,61 @@ Y_UNIT_TEST_SUITE(TFlushRequestTest)
UNIT_ASSERT_VALUES_EQUAL(43, response.FlushFailed[1]);
UNIT_ASSERT_EQUAL_C(route, response.Route, response.Route.DebugPrint());
}
+
+ Y_UNIT_TEST_F(CooldownDelaysFlushRequest, TBaseFixture)
+ {
+ Init();
+
+ // When the oracle reports a non-zero cooldown, the flush request must
+ // not be issued immediately. Instead it is scheduled after the
+ // cooldown delay.
+ const auto cooldown = TDuration::MilliSeconds(30);
+ DirectBlockGroup->Oracle.FlushRequestCooldown = cooldown;
+ DirectBlockGroup->Oracle.FlushRequestTimeout = TDuration::Seconds(5);
+
+ THostRoute route{.SourceHostIndex = 1, .DestinationHostIndex = 2};
+ TFlushHint hint;
+ hint.Segments.push_back(TPBufferSegment{
+ .Lsn = 42,
+ .Range = TBlockRange64::WithLength(10, 3)});
+
+ auto flushRequest = std::make_shared<TFlushRequestExecutor>(
+ Runtime->GetActorSystem(0),
+ LogTitle,
+ VChunkConfig,
+ DirectBlockGroup,
+ route,
+ std::move(hint),
+ NWilson::TSpan());
+
+ auto future = flushRequest->GetFuture();
+ flushRequest->Run();
+
+ // No flush request is issued yet - only the cooldown was scheduled.
+ UNIT_ASSERT_VALUES_EQUAL(false, future.HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(0, FlushPromises.size());
+ UNIT_ASSERT_VALUES_EQUAL(1, ScheduledTasks.size());
+ UNIT_ASSERT_VALUES_EQUAL(cooldown, ScheduledTasks[0].Delay);
+
+ // Fire the cooldown timer - the actual flush request is now issued.
+ ScheduledTasks[0].Callback();
+
+ UNIT_ASSERT_VALUES_EQUAL(1, FlushPromises.size());
+ // A request timeout is scheduled once the flush actually starts.
+ UNIT_ASSERT_VALUES_EQUAL(2, ScheduledTasks.size());
+ UNIT_ASSERT_VALUES_EQUAL(
+ DirectBlockGroup->Oracle.FlushRequestTimeout,
+ ScheduledTasks[1].Delay);
+
+ SetFlushResult(TDBGFlushResponse{.Errors{MakeError(S_OK)}}, false);
+
+ UNIT_ASSERT_VALUES_EQUAL(true, future.HasValue());
+ const auto& response = future.GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(1, response.FlushOk.size());
+ UNIT_ASSERT_VALUES_EQUAL(42, response.FlushOk[0]);
+ UNIT_ASSERT_VALUES_EQUAL(0, response.FlushFailed.size());
+ UNIT_ASSERT_EQUAL_C(route, response.Route, response.Route.DebugPrint());
+ }
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.cpp
index 7537786d26f..6ff06638028 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.cpp
@@ -36,6 +36,15 @@ THostMask THostMask::MakeAll(size_t hostCount)
return THostMask((ui32(1) << hostCount) - 1);
}
+// static
+THostMask THostMask::MakeFromRoute(const THostRoute& route)
+{
+ THostMask result;
+ result.Set(route.SourceHostIndex);
+ result.Set(route.DestinationHostIndex);
+ return result;
+}
+
void THostMask::Set(THostIndex host)
{
Y_ABORT_UNLESS(host < MaxHostCount);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.h
index dc2c099e81b..43329162728 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask.h
@@ -34,6 +34,7 @@ public:
static THostMask MakeEmpty();
static THostMask MakeOne(THostIndex host);
static THostMask MakeAll(size_t hostCount);
+ static THostMask MakeFromRoute(const THostRoute& route);
void Set(THostIndex host);
void Reset(THostIndex host);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask_ut.cpp
index f2a8b9a0f56..eec319ae377 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask_ut.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_mask_ut.cpp
@@ -44,6 +44,23 @@ Y_UNIT_TEST_SUITE(THostMaskTest)
UNIT_ASSERT(mask.Get(7));
}
+ Y_UNIT_TEST(ShouldMakeFromRoute)
+ {
+ // A route with distinct source and destination sets both bits.
+ auto mask = THostMask::MakeFromRoute(
+ THostRoute{.SourceHostIndex = 1, .DestinationHostIndex = 4});
+ UNIT_ASSERT_VALUES_EQUAL(2u, mask.Count());
+ UNIT_ASSERT(mask.Get(1));
+ UNIT_ASSERT(mask.Get(4));
+ UNIT_ASSERT(!mask.Get(0));
+
+ // A route where source equals destination collapses to a single bit.
+ auto sameHost = THostMask::MakeFromRoute(
+ THostRoute{.SourceHostIndex = 2, .DestinationHostIndex = 2});
+ UNIT_ASSERT_VALUES_EQUAL(1u, sameHost.Count());
+ UNIT_ASSERT(sameHost.Get(2));
+ }
+
Y_UNIT_TEST(ShouldDoLogicalOps)
{
auto first = THostMask::MakeOne(0).Include(THostMask::MakeOne(1));
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.cpp
index 6f17f870794..c442a6fa92e 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.cpp
@@ -78,6 +78,11 @@ size_t THostStat::GetConsecutiveSuccessCount() const
return ConsecutiveSuccessCount;
}
+size_t THostStat::GetConsecutiveErrorCount() const
+{
+ return ConsecutiveErrorCount;
+}
+
size_t THostStat::InflightCount(EOperation operation) const
{
return InflightByOperation[static_cast<size_t>(operation)];
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.h
index 70ba3ad9fe1..a407b69dec5 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat.h
@@ -57,6 +57,9 @@ public:
// Number of consecutive successful completions since the last error
// (reset to 0 on the first error after a success streak).
[[nodiscard]] size_t GetConsecutiveSuccessCount() const;
+ // Number of consecutive failed completions since the last success
+ // (reset to 0 on the first success after a error streak).
+ [[nodiscard]] size_t GetConsecutiveErrorCount() const;
// Number of currently inflight requests of a given operation type for
// this host (i.e. OnRequest calls without a matching OnSuccess/OnError).
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat_ut.cpp
index e409f2201ef..a0d965d5d18 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat_ut.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/host_stat_ut.cpp
@@ -190,6 +190,37 @@ Y_UNIT_TEST_SUITE(THostStatTest)
UNIT_ASSERT_VALUES_EQUAL(1, errorsInfo.ConsecutiveErrorCount);
}
+ Y_UNIT_TEST(ErrorCountGetterReflectsConsecutiveErrors)
+ {
+ THostStat stat;
+ TInstant now = TInstant::Now();
+
+ // Initially there are no consecutive errors.
+ UNIT_ASSERT_VALUES_EQUAL(0, stat.GetConsecutiveErrorCount());
+
+ // Each error increments the consecutive error count.
+ stat.OnRequest(EOperation::WriteToPBuffer);
+ stat.OnError(now, EOperation::WriteToPBuffer);
+ UNIT_ASSERT_VALUES_EQUAL(1, stat.GetConsecutiveErrorCount());
+
+ stat.OnRequest(EOperation::WriteToPBuffer);
+ stat.OnError(now, EOperation::WriteToPBuffer);
+ UNIT_ASSERT_VALUES_EQUAL(2, stat.GetConsecutiveErrorCount());
+
+ // A success resets the consecutive error count back to zero.
+ stat.OnRequest(EOperation::WriteToPBuffer);
+ stat.OnSuccess(now, TDuration(), EOperation::WriteToPBuffer);
+ UNIT_ASSERT_VALUES_EQUAL(0, stat.GetConsecutiveErrorCount());
+
+ // OnCancelled does not affect the consecutive error count.
+ stat.OnRequest(EOperation::WriteToPBuffer);
+ stat.OnError(now, EOperation::WriteToPBuffer);
+ UNIT_ASSERT_VALUES_EQUAL(1, stat.GetConsecutiveErrorCount());
+ stat.OnRequest(EOperation::WriteToPBuffer);
+ stat.OnCancelled(now, EOperation::WriteToPBuffer);
+ UNIT_ASSERT_VALUES_EQUAL(1, stat.GetConsecutiveErrorCount());
+ }
+
Y_UNIT_TEST(SuccessCountInitiallyZero)
{
THostStat stat;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp
index a277aa74d7c..81a2db7ca01 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.cpp
@@ -14,8 +14,13 @@ namespace NYdb::NBS::NBlockStore::NStorage::NPartitionDirect {
namespace {
+////////////////////////////////////////////////////////////////////////////////
+
constexpr TDuration MinReconnectDelay = TDuration::MilliSeconds(20);
constexpr TDuration MaxReconnectDelay = TDuration::Seconds(10);
+constexpr TDuration FlushRequestCooldownPenalty = TDuration::MilliSeconds(10);
+
+////////////////////////////////////////////////////////////////////////////////
TDuration GetFromConfig(ui64 milliseconds, TDuration defaultValue)
{
@@ -316,6 +321,11 @@ TDuration TOracle::GetReadRequestTimeout() const
return DefaultReadRequestTimeout;
}
+EWriteMode TOracle::GetWriteMode() const
+{
+ return DefaultWriteMode;
+}
+
TDuration TOracle::GetWriteHedgingDelay(THostMask hosts, bool indirect) const
{
TDuration result = GetTimePredictor(
@@ -335,6 +345,25 @@ TDuration TOracle::GetIndirectWriteReplyTimeout() const
return DefaultIndirectWriteReplyTimeout;
}
+TDuration TOracle::GetFlushRequestCooldown(THostMask hosts) const
+{
+ auto cooldown = [&](THostIndex host) -> TDuration
+ {
+ const size_t errorCount =
+ HostStatistics[host].GetConsecutiveErrorCount();
+ if (!errorCount) {
+ return TDuration::Zero();
+ }
+ return errorCount * FlushRequestCooldownPenalty;
+ };
+
+ TDuration result;
+ for (auto host: hosts) {
+ result = Max(result, cooldown(host));
+ }
+ return Min(result, MaxReconnectDelay);
+}
+
TDuration TOracle::GetFlushRequestTimeout() const
{
return DefaultFlushRequestTimeout;
@@ -345,11 +374,6 @@ TDuration TOracle::GetEraseRequestTimeout() const
return DefaultEraseRequestTimeout;
}
-EWriteMode TOracle::GetWriteMode() const
-{
- return DefaultWriteMode;
-}
-
const THostStat& TOracle::GetHostStatistics(THostIndex hostIndex) const
{
return HostStatistics[hostIndex];
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h
index f2100759169..65fb5ab8a11 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle.h
@@ -66,14 +66,19 @@ public:
THostIndex host,
EDataLocation dataLocation) const = 0;
[[nodiscard]] virtual TDuration GetReadRequestTimeout() const = 0;
+
+ [[nodiscard]] virtual EWriteMode GetWriteMode() const = 0;
[[nodiscard]] virtual TDuration GetWriteHedgingDelay(
THostMask hosts,
bool indirect) const = 0;
[[nodiscard]] virtual TDuration GetWriteRequestTimeout() const = 0;
[[nodiscard]] virtual TDuration GetIndirectWriteReplyTimeout() const = 0;
+
+ [[nodiscard]] virtual TDuration GetFlushRequestCooldown(
+ THostMask hosts) const = 0;
[[nodiscard]] virtual TDuration GetFlushRequestTimeout() const = 0;
+
[[nodiscard]] virtual TDuration GetEraseRequestTimeout() const = 0;
- [[nodiscard]] virtual EWriteMode GetWriteMode() const = 0;
[[nodiscard]] virtual const THostStat& GetHostStatistics(
THostIndex hostIndex) const = 0;
@@ -92,6 +97,7 @@ public:
void Think(TInstant now);
+ // IOracle implementation
void OnRequestStarted(
THostIndex hostIndex,
EOperation operation,
@@ -123,14 +129,19 @@ public:
THostIndex host,
EDataLocation dataLocation) const override;
[[nodiscard]] TDuration GetReadRequestTimeout() const override;
+
+ [[nodiscard]] EWriteMode GetWriteMode() const override;
[[nodiscard]] TDuration GetWriteHedgingDelay(
THostMask hosts,
bool indirect) const override;
[[nodiscard]] TDuration GetWriteRequestTimeout() const override;
[[nodiscard]] TDuration GetIndirectWriteReplyTimeout() const override;
+
+ [[nodiscard]] TDuration GetFlushRequestCooldown(
+ THostMask hosts) const override;
[[nodiscard]] TDuration GetFlushRequestTimeout() const override;
+
[[nodiscard]] TDuration GetEraseRequestTimeout() const override;
- [[nodiscard]] EWriteMode GetWriteMode() const override;
[[nodiscard]] const THostStat& GetHostStatistics(
THostIndex hostIndex) const override;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle_ut.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle_ut.cpp
index 5955cd35d4d..aef44ab809d 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle_ut.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/oracle_ut.cpp
@@ -541,6 +541,121 @@ Y_UNIT_TEST_SUITE(TOracle)
TDuration::MilliSeconds(300),
oracle.GetWriteHedgingDelay(THostMask::MakeOne(0), true));
}
+
+ Y_UNIT_TEST(GetFlushRequestCooldownReturnsZeroWithoutErrors)
+ {
+ NProto::TStorageServiceConfig rawConfig;
+ auto storageConfig = std::make_shared<TStorageConfig>(rawConfig);
+
+ TOracle oracle(storageConfig, nullptr);
+
+ // No errors recorded on any host -> no cooldown.
+ UNIT_ASSERT_VALUES_EQUAL(
+ TDuration::Zero(),
+ oracle.GetFlushRequestCooldown(THostMask::MakeAll(5)));
+ }
+
+ Y_UNIT_TEST(GetFlushRequestCooldownScalesWithConsecutiveErrors)
+ {
+ NProto::TStorageServiceConfig rawConfig;
+ auto storageConfig = std::make_shared<TStorageConfig>(rawConfig);
+
+ TOracle oracle(storageConfig, nullptr);
+ const auto now = TInstant::Now();
+
+ // Each consecutive error adds a 10ms penalty on host 0.
+ for (size_t errorCount = 1; errorCount <= 3; ++errorCount) {
+ oracle.OnRequestStarted(0, EOperation::WriteToPBuffer, now);
+ oracle.OnRequestFailed(0, EOperation::WriteToPBuffer, now);
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TDuration::MilliSeconds(10 * errorCount),
+ oracle.GetFlushRequestCooldown(THostMask::MakeOne(0)));
+ }
+
+ // A single success resets the consecutive error count back to zero.
+ oracle.OnRequestStarted(0, EOperation::WriteToPBuffer, now);
+ oracle.OnRequestSucceeded(
+ 0,
+ EOperation::WriteToPBuffer,
+ now,
+ TDuration());
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ TDuration::Zero(),
+ oracle.GetFlushRequestCooldown(THostMask::MakeOne(0)));
+ }
+
+ Y_UNIT_TEST(GetFlushRequestCooldownReturnsMaxAcrossHosts)
+ {
+ NProto::TStorageServiceConfig rawConfig;
+ auto storageConfig = std::make_shared<TStorageConfig>(rawConfig);
+
+ TOracle oracle(storageConfig, nullptr);
+ const auto now = TInstant::Now();
+
+ // Host 1: two consecutive errors -> 20ms.
+ for (size_t i = 0; i < 2; ++i) {
+ oracle.OnRequestStarted(1, EOperation::WriteToPBuffer, now);
+ oracle.OnRequestFailed(1, EOperation::WriteToPBuffer, now);
+ }
+
+ // Host 3: four consecutive errors -> 40ms.
+ for (size_t i = 0; i < 4; ++i) {
+ oracle.OnRequestStarted(3, EOperation::WriteToPBuffer, now);
+ oracle.OnRequestFailed(3, EOperation::WriteToPBuffer, now);
+ }
+
+ // A mask covering both hosts must return the maximum cooldown.
+ UNIT_ASSERT_VALUES_EQUAL(
+ TDuration::MilliSeconds(40),
+ oracle.GetFlushRequestCooldown(
+ THostMask::MakeOne(1).Include(THostMask::MakeOne(3))));
+
+ // A mask restricted to the less-penalized host returns its own value.
+ UNIT_ASSERT_VALUES_EQUAL(
+ TDuration::MilliSeconds(20),
+ oracle.GetFlushRequestCooldown(THostMask::MakeOne(1)));
+
+ // A mask covering only error-free hosts returns zero.
+ UNIT_ASSERT_VALUES_EQUAL(
+ TDuration::Zero(),
+ oracle.GetFlushRequestCooldown(
+ THostMask::MakeOne(0).Include(THostMask::MakeOne(4))));
+ }
+
+ Y_UNIT_TEST(GetFlushRequestCooldownIsClampedToMaxReconnectDelay)
+ {
+ NProto::TStorageServiceConfig rawConfig;
+ auto storageConfig = std::make_shared<TStorageConfig>(rawConfig);
+
+ TOracle oracle(storageConfig, nullptr);
+ const auto now = TInstant::Now();
+
+ // The cooldown grows by 10ms per consecutive error and is capped at
+ // MaxReconnectDelay (10s). 10s / 10ms == 1000 errors reach the cap
+ // exactly; any additional error must not exceed it.
+ const auto maxReconnectDelay = TDuration::Seconds(10);
+ for (size_t i = 0; i < 1000; ++i) {
+ oracle.OnRequestStarted(0, EOperation::WriteToPBuffer, now);
+ oracle.OnRequestFailed(0, EOperation::WriteToPBuffer, now);
+ }
+
+ // Exactly at the cap.
+ UNIT_ASSERT_VALUES_EQUAL(
+ maxReconnectDelay,
+ oracle.GetFlushRequestCooldown(THostMask::MakeOne(0)));
+
+ // Far beyond the cap - still clamped.
+ for (size_t i = 0; i < 500; ++i) {
+ oracle.OnRequestStarted(0, EOperation::WriteToPBuffer, now);
+ oracle.OnRequestFailed(0, EOperation::WriteToPBuffer, now);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(
+ maxReconnectDelay,
+ oracle.GetFlushRequestCooldown(THostMask::MakeOne(0)));
+ }
}
////////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.cpp
index 616189f8841..08ff001725c 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.cpp
@@ -89,6 +89,16 @@ ui32 TVChunkConfig::GetVChunkIndex() const
return VChunkIndex;
}
+void TVChunkConfig::SetDBGIndex(ui32 dbgIndex)
+{
+ DBGIndex = dbgIndex;
+}
+
+ui32 TVChunkConfig::GetDBGIndex() const
+{
+ return DBGIndex;
+}
+
void TVChunkConfig::EnableHost(THostIndex hostIndex)
{
EnabledHosts.Set(hostIndex);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.h
index 6a0e5374c3c..4069fee3da6 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/model/vchunk_config.h
@@ -25,6 +25,9 @@ public:
[[nodiscard]] size_t GetHostCount() const;
[[nodiscard]] ui32 GetVChunkIndex() const;
+ void SetDBGIndex(ui32 dbgIndex);
+ [[nodiscard]] ui32 GetDBGIndex() const;
+
// Enables the host to work. If the total count of ddisks is not enough to
// reach the quorum, then can make a promotion for ddisk.
void EnableHost(THostIndex hostIndex);
@@ -73,6 +76,7 @@ public:
private:
size_t HostCount = 0;
+ ui32 DBGIndex = 0;
ui32 VChunkIndex = 0;
THostRoles PBufferHosts;
THostRoles DDiskHosts;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.cpp
index 7bc57d17bee..d0c147f6ecc 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.cpp
@@ -12,6 +12,16 @@
namespace NYdb::NBS::NBlockStore::NStorage::NPartitionDirect {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+constexpr auto SlowRequestTime = TDuration::Seconds(1);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+
////////////////////////////////////////////////////////////////////////////////
TReadSingleLocationRequestExecutor::TReadSingleLocationRequestExecutor(
@@ -56,6 +66,7 @@ TReadSingleLocationRequestExecutor::~TReadSingleLocationRequestExecutor()
void TReadSingleLocationRequestExecutor::Run()
{
+ StartAt = TInstant::Now();
ScheduleRequestTimeout();
StartReading();
@@ -64,11 +75,8 @@ void TReadSingleLocationRequestExecutor::Run()
TString TReadSingleLocationRequestExecutor::Print()
{
TStringBuilder result;
- result << LogTitle.GetWithTime();
- result << "c:" << ReadHint.HostMask.Print();
- result << ",r:" << Requested.Print();
- result << ",f:" << Failed.Print();
- result << (Promise.IsReady() ? ",Replied" : ",NotReplied");
+ result << LogTitle.GetWithTime() << " " << ExtendedDebugState() << " "
+ << (Promise.IsReady() ? ",Replied" : ",NotReplied");
return result;
}
@@ -90,18 +98,14 @@ void TReadSingleLocationRequestExecutor::StartReading()
auto host = candidates.First();
if (!host) {
if (Requested == Failed) {
- Reply(MakeError(
- E_REJECTED,
- TStringBuilder() << "Can't read. r:" << Requested.Print()
- << ",f:" << Failed.Print()));
+ Reply(MakeError(E_REJECTED, ExtendedDebugState()));
} else {
LOG_DEBUG(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s Read started from all available hosts. r:%s,f:%s",
+ "%s Read started from all available hosts. %s",
LogTitle.GetWithTime().c_str(),
- Requested.Print().c_str(),
- Failed.Print().c_str());
+ ExtendedDebugState().c_str());
}
return;
}
@@ -126,11 +130,9 @@ void TReadSingleLocationRequestExecutor::StartReading()
PrintHostIndex(*host).c_str(),
tryCount);
- auto onReadResponse = [self = shared_from_this(), host = *host] //
- (const NThreading::TFuture<TDBGReadBlocksResponse>& f)
- {
- self->OnReadResponse(host, f.GetValue());
- };
+ ScheduleHedging(DirectBlockGroup->GetOracle()->GetReadHedgingDelay(
+ *host,
+ ReadHint.Lsn == 0 ? EDataLocation::DDisk : EDataLocation::PBuffer));
auto future = fromDDisk ? DirectBlockGroup->ReadBlocksFromDDisk(
VChunkConfig.GetVChunkIndex(),
@@ -145,11 +147,12 @@ void TReadSingleLocationRequestExecutor::StartReading()
ReadHint.VChunkRange,
Request->Sglist,
TraceId);
- future.Subscribe(std::move(onReadResponse));
-
- ScheduleHedging(DirectBlockGroup->GetOracle()->GetReadHedgingDelay(
- *host,
- ReadHint.Lsn == 0 ? EDataLocation::DDisk : EDataLocation::PBuffer));
+ future.Subscribe(
+ [self = shared_from_this(), host = *host] //
+ (const NThreading::TFuture<TDBGReadBlocksResponse>& f)
+ {
+ self->OnReadResponse(host, f.GetValue()); //
+ });
}
void TReadSingleLocationRequestExecutor::OnReadResponse(
@@ -165,12 +168,11 @@ void TReadSingleLocationRequestExecutor::OnReadResponse(
LOG_WARN(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s %s: %s, r:%s,f:%s",
+ "%s %s: %s, %s",
LogTitle.GetWithTime().c_str(),
PrintHostIndex(host).c_str(),
FormatError(response.Error).c_str(),
- Requested.Print().c_str(),
- Failed.Print().c_str());
+ ExtendedDebugState().c_str());
StartReading();
}
@@ -181,19 +183,31 @@ void TReadSingleLocationRequestExecutor::Reply(NProto::TError error)
return;
}
+ const auto duration = TInstant::Now() - StartAt;
+ if (duration > SlowRequestTime) {
+ LOG_INFO(
+ *ActorSystem,
+ NKikimrServices::NBS_PARTITION,
+ "%s [?] Slow request %s",
+ LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str());
+ }
+
if (HasError(error)) {
LOG_ERROR(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s [!] Reply error: %s",
+ "%s [!] Reply error: %s %s",
LogTitle.GetWithTime().c_str(),
- FormatError(error).c_str());
+ FormatError(error).c_str(),
+ ExtendedDebugState().c_str());
} else {
LOG_DEBUG(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s Reply OK",
- LogTitle.GetWithTime().c_str());
+ "%s Reply OK %s",
+ LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str());
}
ReadHint.Lock.Disarm();
@@ -211,8 +225,9 @@ void TReadSingleLocationRequestExecutor::ScheduleHedging(TDuration hedgingDelay)
LOG_DEBUG(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s Schedule OnHedgingTimeout %s",
+ "%s Schedule OnHedgingTimeout %s %s",
LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str(),
FormatDuration(hedgingDelay).c_str());
DirectBlockGroup->Schedule(
@@ -259,7 +274,7 @@ void TReadSingleLocationRequestExecutor::OnHedgingTimeout()
NKikimrServices::NBS_PARTITION,
"%s OnHedgingTimeout %s",
LogTitle.GetWithTime().c_str(),
- Print().c_str());
+ ExtendedDebugState().c_str());
const bool allRetriesAreSpent = ReadHint.HostMask == Requested;
if (!allRetriesAreSpent) {
@@ -276,12 +291,22 @@ void TReadSingleLocationRequestExecutor::OnRequestTimeout()
LOG_WARN(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s OnRequestTimeout.",
- LogTitle.GetWithTime().c_str());
+ "%s OnRequestTimeout. %s",
+ LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str());
Reply(MakeError(E_TIMEOUT, "Request timeout"));
}
+TString TReadSingleLocationRequestExecutor::ExtendedDebugState() const
+{
+ TStringBuilder result;
+ result << "a:" << ReadHint.HostMask.Print();
+ result << ",r:" << Requested.Print();
+ result << ",f:" << Failed.Print();
+ return result;
+}
+
////////////////////////////////////////////////////////////////////////////////
} // namespace NYdb::NBS::NBlockStore::NStorage::NPartitionDirect
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.h
index 9546247e43b..292da6385e1 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/read_request_single_location.h
@@ -54,6 +54,8 @@ private:
void OnHedgingTimeout();
void OnRequestTimeout();
+ TString ExtendedDebugState() const;
+
NActors::TActorSystem const* ActorSystem;
const TChildLogTitle LogTitle;
const TVChunkConfig VChunkConfig;
@@ -63,6 +65,7 @@ private:
const NWilson::TTraceId TraceId;
const TDuration RequestTimeout;
+ TInstant StartAt;
TReadRangeHint ReadHint;
THostMask Requested;
THostMask Failed;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/region.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/region.cpp
index 0a8c5843ecf..209953e944b 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/region.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/region.cpp
@@ -44,11 +44,12 @@ TRegion::TRegion(
counters->GetSubgroup("vchunk", ToString(vChunkIndex));
const auto* persisted = vChunkConfigs.FindPtr(vChunkIndex);
- const auto vChunkConfig = persisted ? *persisted
- : TVChunkConfig::MakeDefault(
- vChunkIndex,
- DirectBlockGroupHostCount,
- DefaultPrimaryCount);
+ auto vChunkConfig = persisted ? *persisted
+ : TVChunkConfig::MakeDefault(
+ vChunkIndex,
+ DirectBlockGroupHostCount,
+ DefaultPrimaryCount);
+ vChunkConfig.SetDBGIndex(dbgIndex);
Y_ABORT_UNLESS(vChunkConfig.IsValid());
Y_ABORT_UNLESS(vChunkConfig.GetVChunkIndex() == vChunkIndex);
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp
index dd780c78830..6622d221149 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/vchunk.cpp
@@ -40,7 +40,10 @@ TVChunk::TVChunk(
, BlockSize(DefaultBlockSize)
, BlocksCount(vChunkSize / BlockSize)
, SyncRequestsBatchSize(syncRequestsBatchSize)
- , LogTitle{GetCycleCount(), TLogTitle::TVChunk{.VChunkIndex = vChunkConfig.GetVChunkIndex()}}
+ , LogTitle{GetCycleCount(), TLogTitle::TVChunk{
+ .DBGIndex = vChunkConfig.GetDBGIndex(),
+ .VChunkIndex = vChunkConfig.GetVChunkIndex()
+ }}
, VChunkConfig(vChunkConfig)
, BlocksDirtyMap(VChunkConfig, BlockSize, BlocksCount)
, Counters(std::move(counters))
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.cpp b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.cpp
index ccb6cdfcce8..46edb5ba795 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.cpp
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.cpp
@@ -13,6 +13,16 @@
namespace NYdb::NBS::NBlockStore::NStorage::NPartitionDirect {
+namespace {
+
+////////////////////////////////////////////////////////////////////////////////
+
+constexpr auto SlowRequestTime = TDuration::Seconds(1);
+
+////////////////////////////////////////////////////////////////////////////////
+
+} // namespace
+
////////////////////////////////////////////////////////////////////////////////
TWriteRequestExecutor::TWriteRequestExecutor(
@@ -53,6 +63,7 @@ TWriteRequestExecutor::~TWriteRequestExecutor()
void TWriteRequestExecutor::Run()
{
+ StartAt = TInstant::Now();
Bundle->GetSpan().Event("Run");
const auto hosts = VChunkConfig.GetDesiredPBuffers();
@@ -366,6 +377,17 @@ void TWriteRequestExecutor::Reply(NProto::TError error)
Y_ABORT_IF(IsReplied, "TWriteRequestExecutor::Reply called twice");
IsReplied = true;
+ const auto duration = TInstant::Now() - StartAt;
+ if (duration > SlowRequestTime) {
+ LOG_INFO(
+ *ActorSystem,
+ NKikimrServices::NBS_PARTITION,
+ "%s [?] Slow request %s %s",
+ LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str(),
+ FormatDuration(duration).c_str());
+ }
+
if (HasError(error)) {
LOG_ERROR(
*ActorSystem,
@@ -378,8 +400,9 @@ void TWriteRequestExecutor::Reply(NProto::TError error)
LOG_DEBUG(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s Reply OK.",
- LogTitle.GetWithTime().c_str());
+ "%s Reply OK. %s",
+ LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str());
}
Bundle->Reply(
@@ -414,8 +437,9 @@ void TWriteRequestExecutor::ScheduleHedging(TDuration hedgingDelay)
LOG_DEBUG(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s Schedule OnHedgingTimeout() %s",
+ "%s Schedule OnHedgingTimeout %s %s",
LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str(),
FormatDuration(hedgingDelay).c_str());
DirectBlockGroup->Schedule(
@@ -477,8 +501,9 @@ void TWriteRequestExecutor::OnRequestTimeout()
LOG_WARN(
*ActorSystem,
NKikimrServices::NBS_PARTITION,
- "%s Request timeout.",
- LogTitle.GetWithTime().c_str());
+ "%s Request timeout. %s",
+ LogTitle.GetWithTime().c_str(),
+ ExtendedDebugState().c_str());
ReplyOrNotifyBelated(MakeError(E_TIMEOUT, "Write request timeout"), {});
}
@@ -514,8 +539,8 @@ THostMask TWriteRequestExecutor::GetRunningDirectWrites() const
TString TWriteRequestExecutor::ExtendedDebugState() const
{
TStringBuilder result;
- result << "dr:" << RequestedDirectWrites.Print();
- result << " ir:" << IndirectCoordinator.Print()
+ result << "d:" << RequestedDirectWrites.Print();
+ result << " i:" << IndirectCoordinator.Print()
<< RequestedIndirectWrites.Print();
result << " c:" << CompletedWrites.Print();
result << " f:" << FailedWrites.Print();
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.h b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.h
index c12d545375c..7cd6e4902c6 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/partition_direct/write_request.h
@@ -79,6 +79,7 @@ private:
const TDuration RequestTimeout;
const TDuration IndirectWriteReplyTimeout;
+ TInstant StartAt;
THostMask IndirectCoordinator;
THostMask RequestedIndirectWrites;
THostMask RequestedDirectWrites;
diff --git a/ydb/core/nbs/cloud/blockstore/libs/storage/storage_transport/ic_storage_transport_actor.h b/ydb/core/nbs/cloud/blockstore/libs/storage/storage_transport/ic_storage_transport_actor.h
index 282d7688b1c..ca854100fbd 100644
--- a/ydb/core/nbs/cloud/blockstore/libs/storage/storage_transport/ic_storage_transport_actor.h
+++ b/ydb/core/nbs/cloud/blockstore/libs/storage/storage_transport/ic_storage_transport_actor.h
@@ -110,7 +110,6 @@ private:
void HandleBatchErasePersistentBuffer(
const TEvTransportPrivate::TEvBatchEraseFromPBuffer::TPtr& ev,
const NActors::TActorContext& ctx);
-
void HandleBarrierErasePersistentBuffer(
const TEvTransportPrivate::TEvBarrierEraseFromPBuffer::TPtr& ev,
const NActors::TActorContext& ctx);