diff options
31 files changed, 223 insertions, 165 deletions
diff --git a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp index b8cc467fae2..c57897a8759 100644 --- a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp +++ b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp @@ -93,8 +93,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { , Controller(NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>()) { Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); Controller->SetExpectedShardsCount(ShardsCount); - Controller->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - Controller->SetReadTimeoutClean(TDuration::Seconds(1)); + Controller->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1)); Tests::NCommon::TLoggerInit(Kikimr).SetComponents({ NKikimrServices::TX_COLUMNSHARD }, "CS").Initialize(); @@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { } void WaitNormalization() { - Controller->SetReadTimeoutClean(TDuration::Seconds(1)); + Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1)); Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Force); const auto start = TInstant::Now(); while (!Controller->IsTrivialLinks() && TInstant::Now() - start < TDuration::Seconds(30)) { @@ -120,11 +120,11 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { } AFL_VERIFY(Controller->IsTrivialLinks()); Controller->CheckInvariants(); - Controller->SetReadTimeoutClean(TDuration::Minutes(5)); + Controller->SetOverrideReadTimeoutClean(TDuration::Minutes(5)); } void Execute(const ui64 destinationIdx, const std::vector<ui64>& sourceIdxs, const bool move, const NOlap::TSnapshot& snapshot, const std::set<ui64>& pathIdxs) { - Controller->SetReadTimeoutClean(TDuration::Seconds(1)); + Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1)); AFL_VERIFY(destinationIdx < ShardIds.size()); const ui64 destination = ShardIds[destinationIdx]; std::vector<ui64> sources; @@ -192,7 +192,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { CSTransferStatus->Reset(); AFL_VERIFY(!Controller->IsTrivialLinks()); Controller->CheckInvariants(); - Controller->SetReadTimeoutClean(TDuration::Minutes(5)); + Controller->SetOverrideReadTimeoutClean(TDuration::Minutes(5)); } }; Y_UNIT_TEST(BlobsSharingSplit1_1) { @@ -318,8 +318,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { void Execute() { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); TLocalHelper(Kikimr).SetShardingMethod(ShardingType).CreateTestOlapTable("olapTable", "olapStore", 24, 4); diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp index feff93803fc..7699cd8d8b6 100644 --- a/ydb/core/kqp/ut/olap/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp @@ -17,8 +17,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { TKikimrRunner kikimr(settings); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); TLocalHelper(kikimr).CreateTestOlapTable(); @@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { TKikimrRunner kikimr(settings); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); TLocalHelper(kikimr).CreateTestOlapTable(); auto tableClient = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp index 4f6c90056e8..c1fcab4be0f 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp @@ -21,10 +21,10 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { class TOlapStatsController : public NYDBTest::NColumnShard::TController { public: - TDuration GetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override { + TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override { return TDuration::MilliSeconds(10); } - TDuration GetStatsReportInterval(const TDuration /*defaultValue*/) const override { + TDuration DoGetStatsReportInterval(const TDuration /*defaultValue*/) const override { return TDuration::MilliSeconds(10); } }; diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 090f5489105..180f5a7ebb1 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -2695,8 +2695,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) { TLocalHelper testHelper(kikimr); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); diff --git a/ydb/core/kqp/ut/olap/sparsed_ut.cpp b/ydb/core/kqp/ut/olap/sparsed_ut.cpp index 1b5011128de..88bc081fa35 100644 --- a/ydb/core/kqp/ut/olap/sparsed_ut.cpp +++ b/ydb/core/kqp/ut/olap/sparsed_ut.cpp @@ -95,7 +95,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { void Execute() { CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - CSController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); + CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); Tests::NCommon::TLoggerInit(Kikimr).Initialize(); TTypedLocalHelper helper("Utf8", Kikimr); diff --git a/ydb/core/kqp/ut/olap/write_ut.cpp b/ydb/core/kqp/ut/olap/write_ut.cpp index ac63da37bc7..6293899abcd 100644 --- a/ydb/core/kqp/ut/olap/write_ut.cpp +++ b/ydb/core/kqp/ut/olap/write_ut.cpp @@ -15,7 +15,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { Y_UNIT_TEST(TierDraftsGC) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>(); csController->SetIndexWriteControllerEnabled(false); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters(); auto settings = TKikimrSettings() @@ -50,7 +50,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { Y_UNIT_TEST(TierDraftsGCWithRestart) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>(); csController->SetIndexWriteControllerEnabled(false); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1000)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1000)); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC); Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters(); @@ -133,7 +133,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { Y_UNIT_TEST(WriteDeleteCleanGC) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC); Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters(); @@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); } - csController->SetReadTimeoutClean(TDuration::Zero()); + csController->SetOverrideReadTimeoutClean(TDuration::Zero()); csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC); { const TInstant start = TInstant::Now(); diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 557eaf8b018..7bf3746fabe 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -3265,8 +3265,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto session = Kikimr->GetTableClient().CreateSession().GetValueSync().GetSession(); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); const TString query = Sprintf(R"( diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index a137cb1495d..79dec737eb5 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1553,6 +1553,12 @@ message TColumnShardConfig { optional uint32 MaxInFlightIntervalsOnRequest = 16; optional uint32 MaxReadStaleness_ms = 18 [default = 300000]; + optional uint32 GCIntervalMs = 19 [default = 30000]; + optional uint32 CompactionActualizationLagMs = 20 [default = 1000]; + optional uint32 ActualizationTasksLagMs = 21 [default = 1000]; + optional uint32 LagForCompactionBeforeTieringsMs = 22 [default = 3600000]; + optional uint32 OptimizerFreshnessCheckDurationMs = 23 [default = 300000]; + optional uint32 SmallPortionDetectSizeLimit = 24 [default = 1048576]; // 1 << 20 } message TSchemeShardConfig { diff --git a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp index b4fb522e3d0..5008e3e57c7 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp @@ -309,7 +309,7 @@ std::shared_ptr<NBlobOperations::NBlobStorage::TGCTask> TBlobManager::BuildGCTas return nullptr; } - if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod(TDuration::Seconds(GC_INTERVAL_SECONDS))) { + if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod()) { ACFL_DEBUG("event", "TBlobManager::BuildGCTask skip")("current_gen", CurrentGen)("current_step", CurrentStep)("reason", "too_often"); BlobsManagerCounters.GCCounters.SkipCollectionThrottling->Add(1); return nullptr; diff --git a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h index 90094e62b7a..52e0f573eb6 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h @@ -134,9 +134,6 @@ struct TBlobManagerCounters { // The implementation of BlobManager that hides all GC-related details class TBlobManager : public IBlobManager, public TCommonBlobsTracker { private: - static constexpr ui64 GC_INTERVAL_SECONDS = 30; - -private: using TBlobAddress = NBlobOperations::NBlobStorage::TBlobAddress; class TGCContext; const TTabletId SelfTabletId; diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 0d94deef36c..3a5c969860a 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -178,7 +178,7 @@ void TColumnShard::Handle(TEvPrivate::TEvReadFinished::TPtr& ev, const TActorCon void TColumnShard::Handle(TEvPrivate::TEvPingSnapshotsUsage::TPtr& /*ev*/, const TActorContext& ctx) { if (auto writeTx = InFlightReadsTracker.Ping( - this, NYDBTest::TControllers::GetColumnShardController()->GetPingCheckPeriod(0.6 * GetMaxReadStaleness()), TInstant::Now())) { + this, NYDBTest::TControllers::GetColumnShardController()->GetPingCheckPeriod(), TInstant::Now())) { Execute(writeTx.release(), ctx); } ctx.Schedule(0.3 * GetMaxReadStaleness(), new TEvPrivate::TEvPingSnapshotsUsage()); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 1875a14d976..207554291d7 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -71,9 +71,8 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) , ProgressTxController(std::make_unique<TTxController>(*this)) , StoragesManager(std::make_shared<NOlap::TStoragesManager>(*this)) , DataLocksManager(std::make_shared<NOlap::NDataLocks::TManager>()) - , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod( - TSettings::DefaultPeriodicWakeupActivationPeriod)) - , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval(TSettings::DefaultStatsReportInterval)) + , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod()) + , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval()) , InFlightReadsTracker(StoragesManager, Counters.GetRequestsTracingCounters()) , TablesManager(StoragesManager, info->TabletID) , Subscribers(std::make_shared<NSubscriber::TManager>(*this)) @@ -680,8 +679,8 @@ void TColumnShard::SetupIndexation() { if (InsertTable->GetPathPriorities().size() && InsertTable->GetPathPriorities().rbegin()->first.GetCategory() == NOlap::TPathInfoIndexPriority::EIndexationPriority::PreventOverload) { force = true; } - const ui64 bytesLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationStartBytesLimit(TSettings::GuaranteeIndexationStartBytesLimit); - const TDuration durationLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationInterval(TSettings::GuaranteeIndexationInterval); + const ui64 bytesLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationStartBytesLimit(); + const TDuration durationLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationInterval(); if (!force && InsertTable->GetCountersCommitted().Bytes < bytesLimit && TMonotonic::Now() < BackgroundController.GetLastIndexationInstant() + durationLimit) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("reason", "not_enough_data_and_too_frequency") @@ -1145,8 +1144,7 @@ const NKikimr::NColumnShard::NTiers::TManager* TColumnShard::GetTierManagerPoint } TDuration TColumnShard::GetMaxReadStaleness() { - return NYDBTest::TControllers::GetColumnShardController()->GetReadTimeoutClean( - TDuration::MilliSeconds(AppDataVerified().ColumnShardConfig.GetMaxReadStaleness_ms())); + return NYDBTest::TControllers::GetColumnShardController()->GetReadTimeoutClean(); } } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 63c3289f9c9..9051ab02a1c 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -421,7 +421,7 @@ std::vector<std::shared_ptr<TTTLColumnEngineChanges>> TColumnEngineForLogs::Star TSaverContext saverContext(StoragesManager); NActualizer::TTieringProcessContext context(memoryUsageLimit, saverContext, dataLocksManager, SignalCounters, ActualizationController); - const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetActualizationTasksLag(TDuration::Seconds(1)); + const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetActualizationTasksLag(); for (auto&& i : pathEviction) { auto g = GetGranuleOptional(i.first); if (g) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h index 211f837074a..1ae41c03980 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h @@ -41,13 +41,9 @@ private: CacheFetchingScripts; public: - static const inline ui64 DefaultRejectMemoryIntervalLimit = TGlobalLimits::DefaultRejectMemoryIntervalLimit; - static const inline ui64 DefaultReduceMemoryIntervalLimit = TGlobalLimits::DefaultReduceMemoryIntervalLimit; - static const inline ui64 DefaultReadSequentiallyBufferSize = TGlobalLimits::DefaultReadSequentiallyBufferSize; - - const ui64 ReduceMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetReduceMemoryIntervalLimit(DefaultReduceMemoryIntervalLimit); - const ui64 RejectMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetRejectMemoryIntervalLimit(DefaultRejectMemoryIntervalLimit); - const ui64 ReadSequentiallyBufferSize = DefaultReadSequentiallyBufferSize; + const ui64 ReduceMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetReduceMemoryIntervalLimit(); + const ui64 RejectMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetRejectMemoryIntervalLimit(); + const ui64 ReadSequentiallyBufferSize = TGlobalLimits::DefaultReadSequentiallyBufferSize; ui64 GetProcessMemoryControlId() const { AFL_VERIFY(ProcessMemoryGuard); diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp b/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp index 385f9d818d2..b017464eefe 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp @@ -8,7 +8,7 @@ std::shared_ptr<NKikimr::NOlap::TGranuleMeta> TGranulesStorage::GetGranuleForCom std::map<NStorageOptimizer::TOptimizationPriority, std::shared_ptr<TGranuleMeta>> granulesSorted; ui32 countChecker = 0; std::optional<NStorageOptimizer::TOptimizationPriority> priorityChecker; - const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetCompactionActualizationLag(TDuration::Seconds(1)); + const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetCompactionActualizationLag(); for (auto&& i : Tables) { NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("path_id", i.first); i.second->ActualizeOptimizer(now, actualizationLag); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp index a36e976ed35..36f467a0313 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp @@ -3,8 +3,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { TDuration GetCommonFreshnessCheckDuration() { - static const TDuration CommonFreshnessCheckDuration = TDuration::Seconds(300); - return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(CommonFreshnessCheckDuration); + return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(); } } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h index f83183c04e3..d686fc71911 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h @@ -19,8 +19,6 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { -static const ui64 SmallPortionDetectSizeLimit = 1 << 20; - TDuration GetCommonFreshnessCheckDuration(); class TSimplePortionsGroupInfo { @@ -683,7 +681,7 @@ private: return; } MainPortion->InitRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized, Others.IsEmpty() && currentInstant > MainPortion->RecordSnapshotMax().GetPlanInstant() + - NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings(TDuration::Minutes(60))); + NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings()); } public: TTaskDescription GetTaskDescription() const { @@ -1104,7 +1102,7 @@ public: } void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) { - if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector(SmallPortionDetectSizeLimit)) { + if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector()) { Counters->SmallPortions->RemovePortion(portion); } if (!RemoveBucket(portion)) { @@ -1146,7 +1144,7 @@ public: } void AddPortion(const std::shared_ptr<TPortionInfo>& portion, const TInstant now) { - if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector(SmallPortionDetectSizeLimit)) { + if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector()) { Counters->SmallPortions->AddPortion(portion); AddOther(portion, now); return; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp index baf229b3d1f..2fe68710d80 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp @@ -4,8 +4,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { TDuration GetCommonFreshnessCheckDuration() { - static const TDuration CommonFreshnessCheckDuration = TDuration::Seconds(300); - return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(CommonFreshnessCheckDuration); + return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(); } -} +} // namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp index 9e8d21bb935..bc007d0fff1 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp @@ -2,12 +2,11 @@ #include <ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/optimizer/optimizer.h> #include <ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/logic/one_head/logic.h> #include <ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/logic/slices/logic.h> -#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h> namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { std::shared_ptr<IOptimizationLogic> TOptimizerPlannerConstructor::BuildLogic() const { - const TDuration freshnessCheckDuration = NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(FreshnessCheckDuration); + const TDuration freshnessCheckDuration = NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(); std::shared_ptr<IOptimizationLogic> logic; if (LogicName == "one_head") { logic = std::make_shared<TOneHeadLogic>(freshnessCheckDuration); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h index 8b85a25c087..cabe72ccc2a 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h @@ -1,13 +1,16 @@ #pragma once +#include <ydb/core/protos/config.pb.h> #include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h> #include <ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/logic/abstract/logic.h> +#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h> namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { class TOptimizerPlannerConstructor: public IOptimizerPlannerConstructor { private: YDB_READONLY_DEF(TString, LogicName); - YDB_READONLY(TDuration, FreshnessCheckDuration, TDuration::Seconds(300)); + YDB_READONLY(TDuration, FreshnessCheckDuration, NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration()); + public: static TString GetClassNameStatic() { return "s-buckets"; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp index 5c70d26a38a..ec344a674fd 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp @@ -9,7 +9,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { void TPortionsBucket::RebuildOptimizedFeature(const TInstant currentInstant) const { for (auto&& [_, p] : Portions) { p.MutablePortionInfo().InitRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized, Portions.size() == 1 && currentInstant > p->RecordSnapshotMax().GetPlanInstant() + - NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings(TDuration::Minutes(60)) + NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings() ); } } diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index 0a82451bb7d..cbe12efbd5e 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -433,6 +433,7 @@ std::shared_ptr<NKikimr::NOlap::IStoragesManager> CommonStoragesManager = Initia Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { void WriteLoadRead(const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, const std::vector<NArrow::NTest::TTestColumn>& key) { + TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -528,6 +529,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { void ReadWithPredicates(const std::vector<NArrow::NTest::TTestColumn>& ydbSchema, const std::vector<NArrow::NTest::TTestColumn>& key) { + TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -624,6 +626,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexWriteOverload) { + TTestBasicRuntime runtime; TTestDbWrapper db; auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<TDefaultTestsController>(); TIndexInfo tableInfo = NColumnShard::BuildTableInfo(testColumns, testKey);; @@ -696,10 +699,11 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexTtl) { + TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(testColumns, testKey); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<TDefaultTestsController>(); - csDefaultControllerGuard->SetTasksActualizationLag(TDuration::Zero()); + csDefaultControllerGuard->SetOverrideTasksActualizationLag(TDuration::Zero()); ui64 pathId = 1; ui32 step = 1000; diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp b/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp index a4d916545ea..b275e17f2fd 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp @@ -1,4 +1,26 @@ #include "abstract.h" +#include <ydb/core/tx/columnshard/columnshard_impl.h> + namespace NKikimr::NYDBTest { + +TDuration ICSController::GetGuaranteeIndexationInterval() const { + const TDuration defaultValue = NColumnShard::TSettings::GuaranteeIndexationInterval; + return DoGetGuaranteeIndexationInterval(defaultValue); +} + +TDuration ICSController::GetPeriodicWakeupActivationPeriod() const { + const TDuration defaultValue = NColumnShard::TSettings::DefaultPeriodicWakeupActivationPeriod; + return DoGetPeriodicWakeupActivationPeriod(defaultValue); +} + +TDuration ICSController::GetStatsReportInterval() const { + const TDuration defaultValue = NColumnShard::TSettings::DefaultStatsReportInterval; + return DoGetStatsReportInterval(defaultValue); +} + +ui64 ICSController::GetGuaranteeIndexationStartBytesLimit() const { + const ui64 defaultValue = NColumnShard::TSettings::GuaranteeIndexationStartBytesLimit; + return DoGetGuaranteeIndexationStartBytesLimit(defaultValue); +} } diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index 68a25a123a5..c104590235d 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -4,6 +4,7 @@ #include <ydb/core/tx/columnshard/common/snapshot.h> #include <ydb/core/tx/columnshard/engines/writer/write_controller.h> #include <ydb/core/tx/tiering/snapshot.h> +#include <ydb/core/tx/columnshard/common/limits.h> #include <ydb/library/accessor/accessor.h> #include <ydb/services/metadata/abstract/fetcher.h> @@ -84,13 +85,70 @@ protected: virtual void DoOnDataSharingStarted(const ui64 /*tabletId*/, const TString& /*sessionId*/) { } + virtual TDuration DoGetPingCheckPeriod(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetOverridenGCPeriod(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetCompactionActualizationLag(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetActualizationTasksLag(const TDuration defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetReduceMemoryIntervalLimit(const ui64 defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetRejectMemoryIntervalLimit(const ui64 defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetReadSequentiallyBufferSize(const ui64 defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetSmallPortionSizeDetector(const ui64 defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetReadTimeoutClean(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetGuaranteeIndexationInterval(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetStatsReportInterval(const TDuration defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration defaultValue) const { + return defaultValue; + } + +private: + inline static const NKikimrConfig::TColumnShardConfig DefaultConfig = {}; + + static const NKikimrConfig::TColumnShardConfig& GetConfig() { + if (HasAppData()) { + return AppDataVerified().ColumnShardConfig; + } + return DefaultConfig; + } + public: virtual void OnRequestTracingChanges( const std::set<NOlap::TSnapshot>& /*snapshotsToSave*/, const std::set<NOlap::TSnapshot>& /*snapshotsToRemove*/) { } - virtual TDuration GetPingCheckPeriod(const TDuration def) const { - return def; + TDuration GetPingCheckPeriod() const { + const TDuration defaultValue = 0.6 * GetReadTimeoutClean(); + return DoGetPingCheckPeriod(defaultValue); } virtual bool IsBackgroundEnabled(const EBackground /*id*/) const { @@ -100,15 +158,17 @@ public: using TPtr = std::shared_ptr<ICSController>; virtual ~ICSController() = default; - virtual TDuration GetOverridenGCPeriod(const TDuration def) const { - return def; + TDuration GetOverridenGCPeriod() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetGCIntervalMs()); + return DoGetOverridenGCPeriod(defaultValue); } virtual void OnSelectShardingFilter() { } - virtual TDuration GetCompactionActualizationLag(const TDuration def) const { - return def; + TDuration GetCompactionActualizationLag() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetCompactionActualizationLagMs()); + return DoGetCompactionActualizationLag(defaultValue); } virtual NColumnShard::TBlobPutResult::TPtr OverrideBlobPutResultOnCompaction( @@ -116,25 +176,25 @@ public: return original; } - virtual TDuration GetRemovedPortionLivetime(const TDuration def) const { - return def; - } - - virtual TDuration GetActualizationTasksLag(const TDuration d) const { - return d; + TDuration GetActualizationTasksLag() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetActualizationTasksLagMs()); + return DoGetActualizationTasksLag(defaultValue); } - virtual ui64 GetReduceMemoryIntervalLimit(const ui64 def) const { - return def; + ui64 GetReduceMemoryIntervalLimit() const { + const ui64 defaultValue = NOlap::TGlobalLimits::DefaultReduceMemoryIntervalLimit; + return DoGetReduceMemoryIntervalLimit(defaultValue); } - virtual ui64 GetRejectMemoryIntervalLimit(const ui64 def) const { - return def; + ui64 GetRejectMemoryIntervalLimit() const { + const ui64 defaultValue = NOlap::TGlobalLimits::DefaultRejectMemoryIntervalLimit; + return DoGetRejectMemoryIntervalLimit(defaultValue); } virtual bool NeedForceCompactionBacketsConstruction() const { return false; } - virtual ui64 GetSmallPortionSizeDetector(const ui64 def) const { - return def; + ui64 GetSmallPortionSizeDetector() const { + const ui64 defaultValue = GetConfig().GetSmallPortionDetectSizeLimit(); + return DoGetSmallPortionSizeDetector(defaultValue); } virtual void OnExportFinished() { } @@ -158,8 +218,9 @@ public: virtual void OnMaxValueUsage() { } - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration def) const { - return def; + virtual TDuration GetLagForCompactionBeforeTierings() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetLagForCompactionBeforeTieringsMs()); + return DoGetLagForCompactionBeforeTierings(defaultValue); } void OnTabletInitCompleted(const NColumnShard::TColumnShard& shard) { @@ -188,29 +249,20 @@ public: } virtual void OnIndexSelectProcessed(const std::optional<bool> /*result*/) { } - virtual TDuration GetReadTimeoutClean(const TDuration def) { - return def; + TDuration GetReadTimeoutClean() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetMaxReadStaleness_ms()); + return DoGetReadTimeoutClean(defaultValue); } virtual EOptimizerCompactionWeightControl GetCompactionControl() const { return EOptimizerCompactionWeightControl::Force; } - virtual TDuration GetTTLDefaultWaitingDuration(const TDuration defaultValue) const { - return defaultValue; - } - virtual TDuration GetGuaranteeIndexationInterval(const TDuration defaultValue) const { - return defaultValue; - } - virtual TDuration GetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const { - return defaultValue; - } - virtual TDuration GetStatsReportInterval(const TDuration defaultValue) const { - return defaultValue; - } - virtual ui64 GetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const { - return defaultValue; - } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const { - return defaultValue; + TDuration GetGuaranteeIndexationInterval() const; + TDuration GetPeriodicWakeupActivationPeriod() const; + TDuration GetStatsReportInterval() const; + ui64 GetGuaranteeIndexationStartBytesLimit() const; + TDuration GetOptimizerFreshnessCheckDuration() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetOptimizerFreshnessCheckDurationMs()); + return DoGetOptimizerFreshnessCheckDuration(defaultValue); } virtual void OnTieringModified(const std::shared_ptr<NColumnShard::TTiersManager>& /*tiers*/) { diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h index 031a8f144a9..a8e259877fd 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/controller.h @@ -12,21 +12,21 @@ namespace NKikimr::NYDBTest::NColumnShard { class TController: public TReadOnlyController { private: using TBase = TReadOnlyController; - YDB_ACCESSOR_DEF(std::optional<TDuration>, RequestsTracePingCheckPeriod); - YDB_ACCESSOR_DEF(std::optional<TDuration>, LagForCompactionBeforeTierings); - YDB_ACCESSOR(std::optional<TDuration>, GuaranteeIndexationInterval, TDuration::Zero()); - YDB_ACCESSOR(std::optional<TDuration>, PeriodicWakeupActivationPeriod, std::nullopt); - YDB_ACCESSOR(std::optional<TDuration>, StatsReportInterval, std::nullopt); - YDB_ACCESSOR(std::optional<ui64>, GuaranteeIndexationStartBytesLimit, 0); - YDB_ACCESSOR(std::optional<TDuration>, OptimizerFreshnessCheckDuration, TDuration::Zero()); - YDB_ACCESSOR_DEF(std::optional<TDuration>, CompactionActualizationLag); - YDB_ACCESSOR_DEF(std::optional<TDuration>, TasksActualizationLag); + YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideRequestsTracePingCheckPeriod); + YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideLagForCompactionBeforeTierings); + YDB_ACCESSOR(std::optional<TDuration>, OverrideGuaranteeIndexationInterval, TDuration::Zero()); + YDB_ACCESSOR(std::optional<TDuration>, OverridePeriodicWakeupActivationPeriod, std::nullopt); + YDB_ACCESSOR(std::optional<TDuration>, OverrideStatsReportInterval, std::nullopt); + YDB_ACCESSOR(std::optional<ui64>, OverrideGuaranteeIndexationStartBytesLimit, 0); + YDB_ACCESSOR(std::optional<TDuration>, OverrideOptimizerFreshnessCheckDuration, TDuration::Zero()); + YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideCompactionActualizationLag); + YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideTasksActualizationLag); + YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideReadTimeoutClean); EOptimizerCompactionWeightControl CompactionControl = EOptimizerCompactionWeightControl::Force; YDB_ACCESSOR(std::optional<ui64>, OverrideReduceMemoryIntervalLimit, 1024); YDB_ACCESSOR_DEF(std::optional<ui64>, OverrideRejectMemoryIntervalLimit); - std::optional<TDuration> ReadTimeoutClean; std::optional<ui32> ExpectedShardsCount; THashMap<ui64, const ::NKikimr::NColumnShard::TColumnShard*> ShardActuals; @@ -130,16 +130,16 @@ private: THashSet<TString> SharingIds; protected: virtual ::NKikimr::NColumnShard::TBlobPutResult::TPtr OverrideBlobPutResultOnCompaction(const ::NKikimr::NColumnShard::TBlobPutResult::TPtr original, const NOlap::TWriteActionsCollection& actions) const override; - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration def) const override { - return LagForCompactionBeforeTierings.value_or(def); + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration def) const override { + return OverrideLagForCompactionBeforeTierings.value_or(def); } - virtual TDuration GetPingCheckPeriod(const TDuration def) const override { - return RequestsTracePingCheckPeriod.value_or(def); + virtual TDuration DoGetPingCheckPeriod(const TDuration def) const override { + return OverrideRequestsTracePingCheckPeriod.value_or(def); } - virtual TDuration GetCompactionActualizationLag(const TDuration def) const override { - return CompactionActualizationLag.value_or(def); + virtual TDuration DoGetCompactionActualizationLag(const TDuration def) const override { + return OverrideCompactionActualizationLag.value_or(def); } @@ -148,8 +148,8 @@ protected: return !DisabledBackgrounds.contains(id); } - virtual TDuration GetActualizationTasksLag(const TDuration d) const override { - return TasksActualizationLag.value_or(d); + virtual TDuration DoGetActualizationTasksLag(const TDuration d) const override { + return OverrideTasksActualizationLag.value_or(d); } virtual void DoOnTabletInitCompleted(const ::NKikimr::NColumnShard::TColumnShard& shard) override; @@ -157,23 +157,29 @@ protected: virtual void DoOnAfterGCAction(const ::NKikimr::NColumnShard::TColumnShard& shard, const NOlap::IBlobsGCAction& action) override; virtual bool DoOnWriteIndexComplete(const NOlap::TColumnEngineChanges& changes, const ::NKikimr::NColumnShard::TColumnShard& shard) override; - virtual TDuration GetGuaranteeIndexationInterval(const TDuration defaultValue) const override { - return GuaranteeIndexationInterval.value_or(defaultValue); + virtual TDuration DoGetGuaranteeIndexationInterval(const TDuration defaultValue) const override { + return OverrideGuaranteeIndexationInterval.value_or(defaultValue); } - TDuration GetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const override { - return PeriodicWakeupActivationPeriod.value_or(defaultValue); + virtual TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const override { + return OverridePeriodicWakeupActivationPeriod.value_or(defaultValue); } - TDuration GetStatsReportInterval(const TDuration defaultValue) const override { - return StatsReportInterval.value_or(defaultValue); + virtual TDuration DoGetStatsReportInterval(const TDuration defaultValue) const override { + return OverrideStatsReportInterval.value_or(defaultValue); } - virtual ui64 GetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const override { - return GuaranteeIndexationStartBytesLimit.value_or(defaultValue); + virtual ui64 DoGetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const override { + return OverrideGuaranteeIndexationStartBytesLimit.value_or(defaultValue); } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const override { - return OptimizerFreshnessCheckDuration.value_or(defaultValue); + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const override { + return OverrideOptimizerFreshnessCheckDuration.value_or(defaultValue); } - virtual TDuration GetReadTimeoutClean(const TDuration def) override { - return ReadTimeoutClean.value_or(def); + virtual TDuration DoGetReadTimeoutClean(const TDuration def) const override { + return OverrideReadTimeoutClean.value_or(def); + } + virtual ui64 DoGetReduceMemoryIntervalLimit(const ui64 def) const override { + return OverrideReduceMemoryIntervalLimit.value_or(def); + } + virtual ui64 DoGetRejectMemoryIntervalLimit(const ui64 def) const override { + return OverrideRejectMemoryIntervalLimit.value_or(def); } virtual EOptimizerCompactionWeightControl GetCompactionControl() const override { return CompactionControl; @@ -190,18 +196,9 @@ protected: } public: - virtual TDuration GetRemovedPortionLivetime(const TDuration /*def*/) const override { - return TDuration::Zero(); - } const TAtomicCounter& GetIndexWriteControllerBrokeCount() const { return IndexWriteControllerBrokeCount; } - virtual ui64 GetReduceMemoryIntervalLimit(const ui64 def) const override { - return OverrideReduceMemoryIntervalLimit.value_or(def); - } - virtual ui64 GetRejectMemoryIntervalLimit(const ui64 def) const override { - return OverrideRejectMemoryIntervalLimit.value_or(def); - } bool IsTrivialLinks() const; TCheckContext CheckInvariants() const; @@ -237,9 +234,6 @@ public: void SetCompactionControl(const EOptimizerCompactionWeightControl value) { CompactionControl = value; } - void SetReadTimeoutClean(const TDuration d) { - ReadTimeoutClean = d; - } bool HasPKSortingOnly() const; diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 9f9e90d0298..c55be945520 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -71,11 +71,11 @@ protected: return EOptimizerCompactionWeightControl::Force; } -public: - virtual TDuration GetOverridenGCPeriod(const TDuration /*def*/) const override { + virtual TDuration DoGetOverridenGCPeriod(const TDuration /*def*/) const override { return TDuration::Zero(); } +public: void WaitCompactions(const TDuration d) const { TInstant start = TInstant::Now(); ui32 compactionsStart = GetCompactionStartedCounter().Val(); diff --git a/ydb/core/tx/columnshard/test_helper/controllers.h b/ydb/core/tx/columnshard/test_helper/controllers.h index 5b48f204e2b..68cd6a1dc4e 100644 --- a/ydb/core/tx/columnshard/test_helper/controllers.h +++ b/ydb/core/tx/columnshard/test_helper/controllers.h @@ -21,24 +21,21 @@ protected: virtual bool NeedForceCompactionBacketsConstruction() const override { return true; } - virtual ui64 GetSmallPortionSizeDetector(const ui64 /*def*/) const override { + virtual ui64 DoGetSmallPortionSizeDetector(const ui64 /*def*/) const override { return SmallSizeDetector.value_or(0); } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { return TDuration::Zero(); } - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { return TDuration::Zero(); } - virtual TDuration GetCompactionActualizationLag(const TDuration /*def*/) const override { + virtual TDuration DoGetCompactionActualizationLag(const TDuration /*def*/) const override { return TDuration::Zero(); } - virtual TDuration GetTTLDefaultWaitingDuration(const TDuration /*defaultValue*/) const override { - return TDuration::Seconds(1); - } public: TWaitCompactionController() { - SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); } ui32 GetFinishedExportsCount() const { diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index a2d4f9e36ab..31d13085a18 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -608,7 +608,7 @@ void TestWriteReadLongTxDup() { void TestWriteRead(bool reboots, const TestTableDescription& table = {}, TString codec = "") { auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<TDefaultTestsController>(); csControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - csControllerGuard->SetReadTimeoutClean(TDuration::Max()); + csControllerGuard->SetOverrideReadTimeoutClean(TDuration::Max()); TTestBasicRuntime runtime; TTester::Setup(runtime); @@ -2592,7 +2592,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TTestBasicRuntime runtime; auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<TDefaultTestsController>(); csDefaultControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); - csDefaultControllerGuard->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csDefaultControllerGuard->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); TTester::Setup(runtime); runtime.SetLogPriority(NKikimrServices::BLOB_CACHE, NActors::NLog::PRI_INFO); @@ -2824,7 +2824,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { PlanCommit(runtime, sender, planStep, txId); } UNIT_ASSERT_EQUAL(cleanupsHappened, 0); - csDefaultControllerGuard->SetRequestsTracePingCheckPeriod(TDuration::Zero()); + csDefaultControllerGuard->SetOverrideRequestsTracePingCheckPeriod(TDuration::Zero()); { auto read = std::make_unique<NColumnShard::TEvPrivate::TEvPingSnapshotsUsage>(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, read.release()); diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 29da71e9a8a..deb7be3d89e 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -174,7 +174,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {}, { auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>(); csControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - csControllerGuard->SetTasksActualizationLag(TDuration::Zero()); + csControllerGuard->SetOverrideTasksActualizationLag(TDuration::Zero()); std::vector<ui64> ts = {1600000000, 1620000000}; ui32 ttlIncSeconds = 1; @@ -526,7 +526,7 @@ std::vector<std::pair<ui32, ui64>> TestTiers(bool reboots, const std::vector<TSt auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>(); csControllerGuard->DisableBackground(NYDBTest::ICSController::EBackground::TTL); - csControllerGuard->SetTasksActualizationLag(TDuration::Zero()); + csControllerGuard->SetOverrideTasksActualizationLag(TDuration::Zero()); TTestBasicRuntime runtime; TTester::Setup(runtime); diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index cd6004c88e2..1681aab2ee7 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -642,8 +642,8 @@ Y_UNIT_TEST_SUITE(TOlap) { runtime.UpdateCurrentTime(TInstant::Now() - TDuration::Seconds(600)); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); // disable stats batching diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index 55fd3a3437e..d80d6aae1c7 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -32,21 +32,15 @@ public: virtual bool NeedForceCompactionBacketsConstruction() const override { return true; } - virtual TDuration GetRemovedPortionLivetime(const TDuration /*def*/) const override { - return TDuration::Zero(); - } - virtual ui64 GetSmallPortionSizeDetector(const ui64 /*def*/) const override { + virtual ui64 DoGetSmallPortionSizeDetector(const ui64 /*def*/) const override { return 0; } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { return TDuration::Zero(); } - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { return TDuration::Zero(); } - virtual TDuration GetTTLDefaultWaitingDuration(const TDuration /*defaultValue*/) const override { - return TDuration::Seconds(1); - } }; |