KIKIMR-19788:correct condition for first indexation start

5 files changed, 21 insertions, 17 deletions

diff --git a/ydb/core/tx/columnshard/background_controller.cpp b/ydb/core/tx/columnshard/background_controller.cpp
index 46181a5cdf7..99010cbaa20 100644
--- a/ydb/core/tx/columnshard/background_controller.cpp
+++ b/ydb/core/tx/columnshard/background_controller.cpp
@@ -53,6 +53,7 @@ void TBackgroundController::CheckDeadlinesIndexation() {
 }
 
 void TBackgroundController::StartIndexing(const NOlap::TColumnEngineChanges& changes) {
+    LastIndexationInstant = TMonotonic::Now();
     Y_ABORT_UNLESS(ActiveIndexationTasks.emplace(changes.GetTaskIdentifier(), TMonotonic::Now()).second);
 }
 
diff --git a/ydb/core/tx/columnshard/background_controller.h b/ydb/core/tx/columnshard/background_controller.h
index a501624661b..fad724213dd 100644
--- a/ydb/core/tx/columnshard/background_controller.h
+++ b/ydb/core/tx/columnshard/background_controller.h
@@ -54,7 +54,7 @@ private:
 
     bool ActiveCleanup = false;
     THashSet<NOlap::TPortionAddress> TtlPortions;
-
+    YDB_READONLY(TMonotonic, LastIndexationInstant, TMonotonic::Zero());
 public:
     THashSet<NOlap::TPortionAddress> GetConflictTTLPortions() const;
     THashSet<NOlap::TPortionAddress> GetConflictCompactionPortions() const;
diff --git a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp
index 0dafcd387d4..d17301acd46 100644
--- a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp
+++ b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp
@@ -65,7 +65,7 @@ public:
     TTxType GetTxType() const override { return TXTYPE_PROGRESS; }
 
     bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
-        LOG_S_DEBUG(TxPrefix() << "execute" << TxSuffix());
+        NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("tx_state", "execute");
         Y_ABORT_UNLESS(Self->ProgressTxInFlight);
 
         size_t removedCount = Self->ProgressTxController->CleanExpiredTxs(txc);
@@ -147,7 +147,7 @@ public:
     }
 
     void Complete(const TActorContext& ctx) override {
-        LOG_S_DEBUG(TxPrefix() << "complete" << TxSuffix());
+        NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("tx_state", "complete");
 
         for (auto& rec : TxEvents) {
             ctx.Send(rec.Target, rec.Event.Release(), 0, rec.Cookie);
@@ -159,8 +159,7 @@ public:
             auto event = res.MakeEvent(Self->TabletID());
             ctx.Send(res.TxInfo.Source, event.release(), 0, res.TxInfo.Cookie);
         }
-
-        Self->EnqueueBackgroundActivities();
+        Self->SetupIndexation();
     }
 
 private:
diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp
index 86e4def369c..da061fff0c7 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.cpp
+++ b/ydb/core/tx/columnshard/columnshard_impl.cpp
@@ -711,7 +711,7 @@ protected:
     }
     virtual bool DoOnError(const TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override {
         AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "DoOnError")("blob_id", range)("status", status.GetErrorMessage())("status_code", status.GetStatus());
-        AFL_VERIFY(false)("blob_id", range);
+        AFL_VERIFY(false)("blob_id", range)("status", status.GetStatus());
         TxEvent->SetPutStatus(NKikimrProto::ERROR);
         TActorContext::AsActorContext().Send(ParentActorId, std::move(TxEvent));
         return false;
@@ -729,15 +729,7 @@ public:
 };
 
 void TColumnShard::StartIndexTask(std::vector<const NOlap::TInsertedData*>&& dataToIndex, const i64 bytesToIndex) {
-    if (bytesToIndex < Limits.MinInsertBytes && dataToIndex.size() < TLimits::MIN_SMALL_BLOBS_TO_INSERT) {
-        if (!bytesToIndex || SkippedIndexations < TSettings::MAX_INDEXATIONS_TO_SKIP) {
-            AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("bytes", bytesToIndex)("blobs_count", dataToIndex.size());
-            ++SkippedIndexations;
-            return;
-        }
-    }
     CSCounters.IndexationInput(bytesToIndex);
-    SkippedIndexations = 0;
 
     std::vector<NOlap::TInsertedData> data;
     data.reserve(dataToIndex.size());
@@ -771,9 +763,20 @@ void TColumnShard::SetupIndexation() {
             ("count", BackgroundController.GetIndexingActiveCount())("insert_overload_size", InsertTable->GetCountersCommitted().Bytes)
             ("indexing_debug", BackgroundController.DebugStringIndexation());
         return;
-    } else {
-        AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "start_indexation_tasks")("insert_overload_size", InsertTable->GetCountersCommitted().Bytes);
     }
+
+    bool force = false;
+    if (InsertTable->GetPathPriorities().size() && InsertTable->GetPathPriorities().rbegin()->first.GetCategory() == NOlap::TPathInfoIndexPriority::EIndexationPriority::PreventOverload) {
+        force = true;
+    }
+    if (!force && InsertTable->GetCountersCommitted().Bytes < TSettings::GuaranteeIndexationStartBytesLimit &&
+        TMonotonic::Now() - BackgroundController.GetLastIndexationInstant() < TSettings::GuaranteeIndexationInterval) {
+        AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("reason", "not_enough_data_and_too_frequency")
+            ("insert_size", InsertTable->GetCountersCommitted().Bytes);
+        return;
+    }
+
+    AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "start_indexation_tasks")("insert_overload_size", InsertTable->GetCountersCommitted().Bytes);
     CSCounters.OnSetupIndexation();
     i64 bytesToIndex = 0;
     std::vector<const NOlap::TInsertedData*> dataToIndex;
diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h
index ba9f090e26c..b5156cd838e 100644
--- a/ydb/core/tx/columnshard/columnshard_impl.h
+++ b/ydb/core/tx/columnshard/columnshard_impl.h
@@ -67,6 +67,8 @@ struct TSettings {
     static constexpr ui32 MAX_ACTIVE_COMPACTIONS = 1;
 
     static constexpr ui32 MAX_INDEXATIONS_TO_SKIP = 16;
+    static constexpr TDuration GuaranteeIndexationInterval = TDuration::Seconds(0);
+    static constexpr i64 GuaranteeIndexationStartBytesLimit = (i64)5 * 1024 * 1024 * 1024;
 
     TControlWrapper BlobWriteGrouppingEnabled;
     TControlWrapper CacheDataAfterIndexing;
@@ -376,7 +378,6 @@ private:
     ui64 OwnerPathId = 0;
     ui64 TabletTxCounter = 0;
     ui64 StatsReportRound = 0;
-    ui32 SkippedIndexations = TSettings::MAX_INDEXATIONS_TO_SKIP; // Force indexation on tablet init
     TString OwnerPath;
 
     TIntrusivePtr<TMediatorTimecastEntry> MediatorTimeCastEntry;