Avoid promoting follower read edge when there are no followers KIKIMR-19846

11 files changed, 69 insertions, 12 deletions

diff --git a/ydb/core/base/tablet.h b/ydb/core/base/tablet.h
index 1c04c09a81..6b18baf0c2 100644
--- a/ydb/core/base/tablet.h
+++ b/ydb/core/base/tablet.h
@@ -54,6 +54,7 @@ struct TEvTablet {
         EvUpdateConfig,
         EvDropLease,
         EvReady,
+        EvFollowerDetached, // from leader to user tablet when a follower is removed
 
         EvCommit = EvBoot + 512,
         EvAux,
@@ -270,6 +271,14 @@ struct TEvTablet {
         {}
     };
 
+    struct TEvFollowerDetached : public TEventLocal<TEvFollowerDetached, EvFollowerDetached> {
+        const ui32 TotalFollowers;
+
+        TEvFollowerDetached(ui32 totalFollowers)
+            : TotalFollowers(totalFollowers)
+        {}
+    };
+
     // tablet
     struct TCommitInfo {
         const ui64 TabletID;
diff --git a/ydb/core/tablet/tablet_sys.cpp b/ydb/core/tablet/tablet_sys.cpp
index 75a103ff8a..5eb00aecf0 100644
--- a/ydb/core/tablet/tablet_sys.cpp
+++ b/ydb/core/tablet/tablet_sys.cpp
@@ -562,6 +562,10 @@ TTablet::EraseFollowerInfo(TMap<TActorId, TLeaderInfo>::iterator followerIt) {
 
     auto retIt = LeaderInfo.erase(followerIt);
 
+    if (UserTablet) {
+        Send(UserTablet, new TEvTablet::TEvFollowerDetached(LeaderInfo.size()));
+    }
+
     TryPumpWaitingForGc();
     TryFinishFollowerSync();
 
diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp
index b38644f5e0..f08d81833f 100644
--- a/ydb/core/tablet_flat/flat_executor.cpp
+++ b/ydb/core/tablet_flat/flat_executor.cpp
@@ -784,14 +784,25 @@ void TExecutor::FollowerAuxUpdate(TString upd) {
     }
 }
 
-void TExecutor::FollowerAttached() {
-    HadFollowerAttached = true;
+void TExecutor::FollowerAttached(ui32 totalFollowers) {
+    Stats->FollowersCount = totalFollowers;
     NeedFollowerSnapshot = true;
 
     if (CurrentStateFunc() != &TThis::StateWork)
         return;
 
     MakeLogSnapshot();
+
+    Owner->OnFollowersCountChanged();
+}
+
+void TExecutor::FollowerDetached(ui32 totalFollowers) {
+    Stats->FollowersCount = totalFollowers;
+
+    if (CurrentStateFunc() != &TThis::StateWork)
+        return;
+
+    Owner->OnFollowersCountChanged();
 }
 
 void TExecutor::FollowerSyncComplete() {
diff --git a/ydb/core/tablet_flat/flat_executor.h b/ydb/core/tablet_flat/flat_executor.h
index 79b2b838bd..dfa6372552 100644
--- a/ydb/core/tablet_flat/flat_executor.h
+++ b/ydb/core/tablet_flat/flat_executor.h
@@ -457,7 +457,6 @@ class TExecutor
     ui64 CompactionReadUniqCounter = 0;
 
     bool LogBatchFlushScheduled = false;
-    bool HadFollowerAttached = false;
     bool NeedFollowerSnapshot = false;
 
     mutable bool HadRejectProbabilityByTxInFly = false;
@@ -654,7 +653,8 @@ public:
 
     void AllowBorrowedGarbageCompaction(ui32 tableId) override;
 
-    void FollowerAttached() override;
+    void FollowerAttached(ui32 totalFollowers) override;
+    void FollowerDetached(ui32 totalFollowers) override;
     void FollowerSyncComplete() override;
     void FollowerGcApplied(ui32 step, TDuration followerSyncDelay) override;
     void FollowerBoot(TEvTablet::TEvFBoot::TPtr &ev, const TActorContext &ctx) override;
diff --git a/ydb/core/tablet_flat/tablet_flat_executed.cpp b/ydb/core/tablet_flat/tablet_flat_executed.cpp
index f18c763e34..de09bd2f6b 100644
--- a/ydb/core/tablet_flat/tablet_flat_executed.cpp
+++ b/ydb/core/tablet_flat/tablet_flat_executed.cpp
@@ -77,9 +77,13 @@ void TTabletExecutedFlat::Handle(TEvTablet::TEvFAuxUpdate::TPtr &ev) {
 }
 
 void TTabletExecutedFlat::Handle(TEvTablet::TEvNewFollowerAttached::TPtr &ev) {
-    Y_UNUSED(ev);
     if (Executor())
-        Executor()->FollowerAttached();
+        Executor()->FollowerAttached(ev->Get()->TotalFollowers);
+}
+
+void TTabletExecutedFlat::Handle(TEvTablet::TEvFollowerDetached::TPtr &ev) {
+    if (Executor())
+        Executor()->FollowerDetached(ev->Get()->TotalFollowers);
 }
 
 void TTabletExecutedFlat::Handle(TEvTablet::TEvFollowerSyncComplete::TPtr &ev) {
@@ -264,6 +268,7 @@ bool TTabletExecutedFlat::HandleDefaultEvents(TAutoPtr<IEventHandle>& ev, const
         hFunc(TEvTablet::TEvFAuxUpdate, Handle);
         hFunc(TEvTablet::TEvFollowerGcApplied, Handle);
         hFunc(TEvTablet::TEvNewFollowerAttached, Handle);
+        hFunc(TEvTablet::TEvFollowerDetached, Handle);
         hFunc(TEvTablet::TEvFollowerSyncComplete, Handle);
         HFuncCtx(TEvTablet::TEvTabletStop, HandleTabletStop, ctx);
         HFuncCtx(TEvTablet::TEvTabletDead, HandleTabletDead, ctx);
@@ -291,6 +296,7 @@ void TTabletExecutedFlat::StateInitImpl(TAutoPtr<IEventHandle>& ev, const TActor
         hFunc(TEvTablet::TEvFAuxUpdate, Handle);
         hFunc(TEvTablet::TEvFollowerGcApplied, Handle);
         hFunc(TEvTablet::TEvNewFollowerAttached, Handle);
+        hFunc(TEvTablet::TEvFollowerDetached, Handle);
         hFunc(TEvTablet::TEvFollowerSyncComplete, Handle);
         HFuncCtx(TEvTablet::TEvTabletStop, HandleTabletStop, ctx);
         HFuncCtx(TEvTablet::TEvTabletDead, HandleTabletDead, ctx);
diff --git a/ydb/core/tablet_flat/tablet_flat_executed.h b/ydb/core/tablet_flat/tablet_flat_executed.h
index 7d697374e7..e1a61e4039 100644
--- a/ydb/core/tablet_flat/tablet_flat_executed.h
+++ b/ydb/core/tablet_flat/tablet_flat_executed.h
@@ -70,6 +70,7 @@ protected:
     void Handle(TEvTablet::TEvFAuxUpdate::TPtr&);
     void Handle(TEvTablet::TEvFollowerGcApplied::TPtr&);
     void Handle(TEvTablet::TEvNewFollowerAttached::TPtr&);
+    void Handle(TEvTablet::TEvFollowerDetached::TPtr&);
     void Handle(TEvTablet::TEvUpdateConfig::TPtr&);
 
     /**
diff --git a/ydb/core/tablet_flat/tablet_flat_executor.cpp b/ydb/core/tablet_flat/tablet_flat_executor.cpp
index 0fab8f98ed..6b1ccd28e4 100644
--- a/ydb/core/tablet_flat/tablet_flat_executor.cpp
+++ b/ydb/core/tablet_flat/tablet_flat_executor.cpp
@@ -64,6 +64,10 @@ namespace NFlatExecutorSetup {
     void ITablet::ReadOnlyLeaseDropped() {
         // nothing by default
     }
+
+    void ITablet::OnFollowersCountChanged() {
+        // nothing by default
+    }
 }
 
 }}
diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h
index ec20755a43..a2f2d425fd 100644
--- a/ydb/core/tablet_flat/tablet_flat_executor.h
+++ b/ydb/core/tablet_flat/tablet_flat_executor.h
@@ -324,6 +324,8 @@ struct TExecutorStats {
     TVector<ui32> YellowMoveChannels;
     TVector<ui32> YellowStopChannels;
 
+    ui32 FollowersCount = 0;
+
     bool IsYellowMoveChannel(ui32 channel) const {
         auto it = std::lower_bound(YellowMoveChannels.begin(), YellowMoveChannels.end(), channel);
         return it != YellowMoveChannels.end() && *it == channel;
@@ -467,6 +469,8 @@ namespace NFlatExecutorSetup {
         virtual TDuration ReadOnlyLeaseDuration();
         virtual void ReadOnlyLeaseDropped();
 
+        virtual void OnFollowersCountChanged();
+
         // create transaction?
     protected:
         ITablet(TTabletStorageInfo *info, const TActorId &tablet)
@@ -503,7 +507,8 @@ namespace NFlatExecutorSetup {
         // next follower incremental update
         virtual void FollowerUpdate(THolder<TEvTablet::TFUpdateBody> upd) = 0;
         virtual void FollowerAuxUpdate(TString upd) = 0;
-        virtual void FollowerAttached() = 0;
+        virtual void FollowerAttached(ui32 totalFollowers) = 0;
+        virtual void FollowerDetached(ui32 totalFollowers) = 0;
         // all known followers are synced to us (called once)
         virtual void FollowerSyncComplete() = 0;
         // all followers had completed log with requested gc-barrier
diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h
index 6d890a3d15..4d71b315e9 100644
--- a/ydb/core/tx/datashard/datashard_impl.h
+++ b/ydb/core/tx/datashard/datashard_impl.h
@@ -1702,6 +1702,7 @@ public:
     bool ReassignChannelsEnabled() const override;
     void OnYellowChannelsChanged() override;
     void OnRejectProbabilityRelaxed() override;
+    void OnFollowersCountChanged() override;
     ui64 GetMemoryUsage() const override;
 
     bool HasPipeServer(const TActorId& pipeServerId);
@@ -1895,6 +1896,9 @@ public:
     bool PromoteFollowerReadEdge(TTransactionContext& txc);
     bool PromoteFollowerReadEdge();
 
+    // Returns true when this shard has potential followers
+    bool HasFollowers() const;
+
     // Returns a suitable row version for performing a transaction
     TRowVersion GetMvccTxVersion(EMvccTxMode mode, TOperation* op = nullptr) const;
 
diff --git a/ydb/core/tx/datashard/datashard_snapshots.cpp b/ydb/core/tx/datashard/datashard_snapshots.cpp
index 037c73ec9f..abf7aa0f99 100644
--- a/ydb/core/tx/datashard/datashard_snapshots.cpp
+++ b/ydb/core/tx/datashard/datashard_snapshots.cpp
@@ -754,9 +754,12 @@ bool TSnapshotManager::RemoveExpiredSnapshots(TInstant now, TTransactionContext&
     }
 
     // Make sure we don't leave followers without any repeatable read version
-    TRowVersion maxRepeatableRead = FollowerReadEdge;
-    if (maxRepeatableRead && !FollowerReadEdgeRepeatable) {
-        maxRepeatableRead = maxRepeatableRead.Prev();
+    TRowVersion maxRepeatableRead = TRowVersion::Max();
+    if (Self->HasFollowers()) {
+        maxRepeatableRead = FollowerReadEdge;
+        if (maxRepeatableRead && !FollowerReadEdgeRepeatable) {
+            maxRepeatableRead = maxRepeatableRead.Prev();
+        }
     }
 
     removed |= AdvanceWatermark(txc.DB, Min(proposed, leastPlanned, leastAcquired, maxWriteVersion, maxRepeatableRead));
diff --git a/ydb/core/tx/datashard/follower_edge.cpp b/ydb/core/tx/datashard/follower_edge.cpp
index 3a563c116a..0663f004fd 100644
--- a/ydb/core/tx/datashard/follower_edge.cpp
+++ b/ydb/core/tx/datashard/follower_edge.cpp
@@ -39,7 +39,7 @@ std::tuple<TRowVersion, bool, ui64> TDataShard::CalculateFollowerReadEdge() cons
 bool TDataShard::PromoteFollowerReadEdge(TTransactionContext& txc) {
     Y_ABORT_UNLESS(!IsFollower());
 
-    if (IsMvccEnabled()) {
+    if (IsMvccEnabled() && HasFollowers()) {
         auto [version, repeatable, waitStep] = CalculateFollowerReadEdge();
 
         if (waitStep) {
@@ -77,7 +77,7 @@ public:
 bool TDataShard::PromoteFollowerReadEdge() {
     Y_ABORT_UNLESS(!IsFollower());
 
-    if (IsMvccEnabled()) {
+    if (IsMvccEnabled() && HasFollowers()) {
         auto [currentEdge, currentRepeatable] = SnapshotManager.GetFollowerReadEdge();
         auto [nextEdge, nextRepeatable, waitStep] = CalculateFollowerReadEdge();
 
@@ -95,4 +95,14 @@ bool TDataShard::PromoteFollowerReadEdge() {
     return false;
 }
 
+void TDataShard::OnFollowersCountChanged() {
+    if (HasFollowers()) {
+        PromoteFollowerReadEdge();
+    }
+}
+
+bool TDataShard::HasFollowers() const {
+    return Executor()->GetStats().FollowersCount > 0;
+}
+
 } // namespace NKikimr::NDataShard