summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov333 <[email protected]>2024-03-12 07:57:23 +0300
committerGitHub <[email protected]>2024-03-12 07:57:23 +0300
commit9e4e82efb3be63a2a9684441e4f66fac9e9dc2b0 (patch)
treea0eb38d9dfe5fed9f73034fc3969beb90daf8a04
parent005091ff6cba7952d811036fdad0d6958a9ba474 (diff)
lighter lock - dont lock whole granule on background process (#2625)
-rw-r--r--ydb/core/tx/columnshard/engines/storage/granule.h4
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h7
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp44
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h145
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp5
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h98
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp213
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h256
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h76
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp5
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h107
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp5
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h524
-rw-r--r--ydb/core/tx/columnshard/engines/storage/optimizer/ya.make2
-rw-r--r--ydb/core/tx/columnshard/engines/storage/storage.cpp6
15 files changed, 72 insertions, 1425 deletions
diff --git a/ydb/core/tx/columnshard/engines/storage/granule.h b/ydb/core/tx/columnshard/engines/storage/granule.h
index 222a661091c..4ce5f6aec26 100644
--- a/ydb/core/tx/columnshard/engines/storage/granule.h
+++ b/ydb/core/tx/columnshard/engines/storage/granule.h
@@ -238,6 +238,10 @@ public:
return OptimizerPlanner->GetUsefulMetric();
}
+ bool IsLockedOptimizer(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const {
+ return OptimizerPlanner->IsLocked(dataLocksManager);
+ }
+
void ActualizeOptimizer(const TInstant currentInstant) const {
if (currentInstant - OptimizerPlanner->GetActualizationInstant() > TDuration::Seconds(1)) {
OptimizerPlanner->Actualize(currentInstant);
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h
index e69d09e827c..31e91cd1bb4 100644
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h
+++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h
@@ -67,7 +67,8 @@ protected:
virtual NJson::TJsonValue DoSerializeToJsonVisual() const {
return NJson::JSON_NULL;
}
-
+ virtual bool DoIsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const = 0;
+
public:
using TFactory = NObjectFactory::TObjectFactory<IOptimizerPlanner, TString>;
IOptimizerPlanner(const ui64 pathId)
@@ -105,6 +106,7 @@ public:
{
}
~TModificationGuard() {
+ TMemoryProfileGuard mGuard("Optimizer");
Owner.ModifyPortions(AddPortions, RemovePortions);
}
};
@@ -119,6 +121,9 @@ public:
}
virtual std::vector<NIndexedReader::TSortableBatchPosition> GetBucketPositions() const = 0;
+ bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const {
+ return DoIsLocked(dataLocksManager);
+ }
NJson::TJsonValue SerializeToJsonVisual() const {
return DoSerializeToJsonVisual();
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp
deleted file mode 100644
index e15eb551ca6..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-#include "blob_size.h"
-#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
-#include <ydb/core/tx/columnshard/engines/changes/general_compaction.h>
-#include <ydb/core/tx/columnshard/data_locks/manager/manager.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-std::shared_ptr<NKikimr::NOlap::TColumnEngineChanges> TBlobsWithSizeLimit::BuildMergeTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const {
- if (PortionsSize > (i64)SizeLimitToMerge || PortionsCount > CountLimitToMerge) {
- i64 currentSum = 0;
- std::vector<std::shared_ptr<TPortionInfo>> portions;
- std::optional<TString> tierName;
- for (auto&& i : Portions) {
- for (auto&& c : i.second) {
- if (locksManager->IsLocked(*c.second)) {
- continue;
- }
- if (c.second->GetMeta().GetTierName() && (!tierName || *tierName < c.second->GetMeta().GetTierName())) {
- tierName = c.second->GetMeta().GetTierName();
- }
- currentSum += c.second->GetBlobBytes();
- portions.emplace_back(c.second);
- if (currentSum > (i64)32 * 1024 * 1024) {
- break;
- }
- }
- if (currentSum > (i64)32 * 1024 * 1024) {
- break;
- }
- }
- if (currentSum > SizeLimitToMerge || PortionsCount > CountLimitToMerge) {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("portions", portions.size())("current_sum", currentSum);
- TSaverContext saverContext(StoragesManager);
- return std::make_shared<NCompaction::TGeneralCompactColumnEngineChanges>(limits.GetSplitSettings(), granule, portions, saverContext);
- } else {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("skip", "not_enough_data");
- }
- } else {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("event", "skip_by_condition");
- }
- return nullptr;
-}
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h
deleted file mode 100644
index 4322d81bad4..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h
+++ /dev/null
@@ -1,145 +0,0 @@
-#pragma once
-#include "counters.h"
-#include <ydb/core/formats/arrow/replace_key.h>
-#include <ydb/library/accessor/accessor.h>
-#include <ydb/core/tx/columnshard/splitter/settings.h>
-#include <ydb/core/tx/columnshard/blobs_action/abstract/write.h>
-#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-class TBlobsWithSizeLimit {
-private:
- YDB_READONLY(ui64, SizeLimit, 0);
- YDB_READONLY(i64, SizeLimitToMerge, (i64)2 * 1024 * 1024);
- YDB_READONLY(i64, CountLimitToMerge, 8);
- YDB_READONLY(i64, PortionsSize, 0);
- YDB_READONLY(i64, PortionsCount, 0);
- std::map<NArrow::TReplaceKey, std::map<ui64, std::shared_ptr<TPortionInfo>>> Portions;
- std::shared_ptr<TCounters> Counters;
- std::shared_ptr<IStoragesManager> StoragesManager;
-public:
- TString DebugString() const {
- return TStringBuilder()
- << "p_count=" << PortionsCount << ";"
- << "p_count_by_key=" << Portions.size() << ";"
- ;
- }
-
- TBlobsWithSizeLimit(const ui64 limit, const std::shared_ptr<TCounters>& counters, const std::shared_ptr<IStoragesManager>& storagesManager)
- : SizeLimit(limit)
- , Counters(counters)
- , StoragesManager(storagesManager)
- {
-
- }
-
- std::shared_ptr<TColumnEngineChanges> BuildMergeTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const;
-
- void AddPortion(const std::shared_ptr<TPortionInfo>& portion) {
- AFL_VERIFY(portion->BlobsBytes() < SizeLimit);
- AFL_VERIFY(Portions[portion->IndexKeyStart()].emplace(portion->GetPortion(), portion).second);
- PortionsSize += portion->BlobsBytes();
- ++PortionsCount;
- Counters->OnAddSmallPortion();
- }
-
- void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) {
- auto it = Portions.find(portion->IndexKeyStart());
- AFL_VERIFY(it != Portions.end());
- AFL_VERIFY(it->second.erase(portion->GetPortion()));
- if (!it->second.size()) {
- Portions.erase(it);
- }
- PortionsSize -= portion->BlobsBytes();
- AFL_VERIFY(PortionsSize >= 0);
- --PortionsCount;
- AFL_VERIFY(PortionsCount >= 0);
- Counters->OnRemoveSmallPortion();
- }
-
- std::optional<TOptimizationPriority> GetWeight() const {
- Y_ABORT_UNLESS(Counters->GetSmallCounts() == PortionsCount);
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("portions_opt_count", PortionsCount)("counter", (ui64)Counters->SmallPortionsByGranule.get());
- if (PortionsSize > SizeLimitToMerge || PortionsCount > CountLimitToMerge) {
- return TOptimizationPriority::Critical(PortionsCount);
- } else {
- return {};
- }
- }
-};
-
-class TBlobsBySize {
-private:
- std::map<ui64, TBlobsWithSizeLimit> BlobsBySizeLimit;
-public:
- TString DebugString() const {
- TStringBuilder sb;
- sb << "(";
- for (auto&& i : BlobsBySizeLimit) {
- sb << "(" << i.first << ":" << i.second.DebugString() << ");";
- }
- sb << ")";
- return sb;
- }
-
- void AddPortion(const std::shared_ptr<TPortionInfo>& portion) {
- auto it = BlobsBySizeLimit.upper_bound(portion->GetBlobBytes());
- if (it != BlobsBySizeLimit.end()) {
- it->second.AddPortion(portion);
- }
- }
-
- void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) {
- auto it = BlobsBySizeLimit.upper_bound(portion->GetBlobBytes());
- if (it != BlobsBySizeLimit.end()) {
- it->second.RemovePortion(portion);
- }
- }
-
- std::optional<TOptimizationPriority> GetWeight() const {
- std::optional<TOptimizationPriority> result;
- for (auto&& i : BlobsBySizeLimit) {
- auto w = i.second.GetWeight();
- if (!w) {
- continue;
- }
- if (!result || *result < *w) {
- result = w;
- }
- }
- return result;
- }
-
- const TBlobsWithSizeLimit* GetMaxWeightLimiter() const {
- std::optional<TOptimizationPriority> resultWeight;
- const TBlobsWithSizeLimit* result = nullptr;
- for (auto&& i : BlobsBySizeLimit) {
- auto w = i.second.GetWeight();
- if (!w) {
- continue;
- }
- if (!resultWeight || *resultWeight < *w) {
- resultWeight = w;
- result = &i.second;
- }
- }
- return result;
- }
-
- std::shared_ptr<TColumnEngineChanges> BuildMergeTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const {
- auto* limiter = GetMaxWeightLimiter();
- if (!limiter) {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("fail", "limiter absent");
- return nullptr;
- }
- return limiter->BuildMergeTask(limits, granule, locksManager);
- }
-
- TBlobsBySize(const std::shared_ptr<TCounters>& counters, const std::shared_ptr<IStoragesManager>& storagesManager) {
-// BlobsBySizeLimit.emplace(512 * 1024, TBlobsWithSizeLimit(512 * 1024, counters, storagesManager));
- BlobsBySizeLimit.emplace(1024 * 1024, TBlobsWithSizeLimit(1024 * 1024, counters, storagesManager));
- }
-};
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp
deleted file mode 100644
index a8c1f6920c1..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-#include "counters.h"
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h
deleted file mode 100644
index e43f9acda67..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h
+++ /dev/null
@@ -1,98 +0,0 @@
-#pragma once
-#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h>
-#include <ydb/core/formats/arrow/replace_key.h>
-#include <ydb/library/accessor/accessor.h>
-#include <ydb/core/tx/columnshard/splitter/settings.h>
-#include <ydb/core/tx/columnshard/counters/engine_logs.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-class TGlobalCounters: public NColumnShard::TCommonCountersOwner {
-private:
- using TBase = NColumnShard::TCommonCountersOwner;
- NMonitoring::TDynamicCounters::TCounterPtr SmallPortionsCount;
- std::shared_ptr<NColumnShard::TIncrementalHistogram> HistogramOverlappedIntervalsCount;
- std::shared_ptr<NColumnShard::TIncrementalHistogram> HistogramOverlappedIntervalsPackedSizeCount;
- std::shared_ptr<NColumnShard::TIncrementalHistogram> HistogramOverlappedIntervalsRawSizeCount;
- std::shared_ptr<NColumnShard::TValueAggregationAgent> SmallPortionsCountByGranule;
-public:
- TGlobalCounters()
- : TBase("IntervalsStorageOptimizer") {
- SmallPortionsCountByGranule = TBase::GetValueAutoAggregations("Granule/SmallPortions/Count");
- SmallPortionsCount = TBase::GetValue("SmallPortions/Count");
-
- const std::set<i64> borders = {0, 1, 2, 4, 8, 16, 32, 64};
- HistogramOverlappedIntervalsCount = std::make_shared<NColumnShard::TIncrementalHistogram>("IntervalsStorageOptimizer", "OverlappedIntervals/Count", "", borders);
- HistogramOverlappedIntervalsPackedSizeCount = std::make_shared<NColumnShard::TIncrementalHistogram>("IntervalsStorageOptimizer", "OverlappedIntervals/Size/Packed", "", borders);
- HistogramOverlappedIntervalsRawSizeCount = std::make_shared<NColumnShard::TIncrementalHistogram>("IntervalsStorageOptimizer", "OverlappedIntervals/Size/Raw", "", borders);
- }
-
- static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildClientSmallPortionsAggregation() {
- return Singleton<TGlobalCounters>()->SmallPortionsCountByGranule->GetClient();
- }
-
- static std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> BuildGuardIntervalsOverlapping() {
- return Singleton<TGlobalCounters>()->HistogramOverlappedIntervalsCount->BuildGuard();
- }
-
- static std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> BuildGuardIntervalsPackedSizeOverlapping() {
- return Singleton<TGlobalCounters>()->HistogramOverlappedIntervalsPackedSizeCount->BuildGuard();
- }
-
- static std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> BuildGuardIntervalsRawSizeOverlapping() {
- return Singleton<TGlobalCounters>()->HistogramOverlappedIntervalsRawSizeCount->BuildGuard();
- }
-
- static std::shared_ptr<NColumnShard::TValueGuard> BuildSmallPortionsGuard() {
- return std::make_shared<NColumnShard::TValueGuard>(Singleton<TGlobalCounters>()->SmallPortionsCount);
- }
-
-};
-
-class TCounters {
-private:
- std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> IntervalsGuard;
- std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> IntervalsPackedSizeGuard;
- std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> IntervalsRawSizeGuard;
- std::shared_ptr<NColumnShard::TValueGuard> SmallPortionsCount;
-public:
- std::shared_ptr<NColumnShard::TValueAggregationClient> SmallPortionsByGranule;
- i64 GetSmallCounts() const {
- return SmallPortionsByGranule->GetValueSimple();
- }
-
- TCounters() {
- IntervalsGuard = TGlobalCounters::BuildGuardIntervalsOverlapping();
- IntervalsPackedSizeGuard = TGlobalCounters::BuildGuardIntervalsPackedSizeOverlapping();
- IntervalsRawSizeGuard = TGlobalCounters::BuildGuardIntervalsRawSizeOverlapping();
- SmallPortionsCount = TGlobalCounters::BuildSmallPortionsGuard();
- SmallPortionsByGranule = TGlobalCounters::BuildClientSmallPortionsAggregation();
- }
-
- void OnRemoveIntervalsCount(const ui32 count, const ui64 rawSize, const ui64 packedSize) {
- IntervalsGuard->Sub(count, 1);
- IntervalsPackedSizeGuard->Sub(count, packedSize);
- IntervalsRawSizeGuard->Sub(count, rawSize);
- }
-
- void OnAddIntervalsCount(const ui32 count, const ui64 rawSize, const ui64 packedSize) {
- IntervalsGuard->Add(count, 1);
- IntervalsPackedSizeGuard->Add(count, packedSize);
- IntervalsRawSizeGuard->Add(count, rawSize);
- }
-
- void OnAddSmallPortion() {
- SmallPortionsCount->Add(1);
- SmallPortionsByGranule->Add(1);
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("add_opt_count", SmallPortionsByGranule->GetValueSimple())("counter", (ui64)SmallPortionsByGranule.get());
- }
-
- void OnRemoveSmallPortion() {
- SmallPortionsCount->Sub(1);
- SmallPortionsByGranule->Remove(1);
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("remove_opt_count", SmallPortionsByGranule->GetValueSimple())("counter", (ui64)SmallPortionsByGranule.get());
- }
-
-};
-
-}
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp
deleted file mode 100644
index d9431021b4d..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp
+++ /dev/null
@@ -1,213 +0,0 @@
-#include "optimizer.h"
-#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
-#include <ydb/core/tx/columnshard/counters/common/owner.h>
-#include <ydb/core/tx/columnshard/counters/engine_logs.h>
-#include <ydb/core/tx/columnshard/data_locks/manager/manager.h>
-#include <ydb/core/tx/columnshard/engines/changes/general_compaction.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-std::vector<std::shared_ptr<TPortionInfo>> TIntervalsOptimizerPlanner::GetPortionsForIntervalStartedIn(const NArrow::TReplaceKey& keyStart, const ui32 countExpectation) const {
- std::vector<std::shared_ptr<TPortionInfo>> result;
- auto it = Positions.find(keyStart);
- AFL_VERIFY(it != Positions.end());
- THashSet<ui64> portionsCurrentlyClosed;
- auto itReverse = make_reverse_iterator(it);
- AFL_VERIFY(itReverse != Positions.rbegin());
- --itReverse;
- for (; itReverse != Positions.rend(); ++itReverse) {
- for (auto&& i : itReverse->second.GetPositions()) {
- if (i.first.GetIsStart()) {
- if (!portionsCurrentlyClosed.erase(i.first.GetPortionId())) {
- result.emplace_back(i.second.GetPortionPtr());
- }
- } else {
- AFL_VERIFY(portionsCurrentlyClosed.emplace(i.first.GetPortionId()).second);
- }
- }
- if (result.size() == countExpectation) {
- return result;
- }
- }
- AFL_VERIFY(false)("result.size()", result.size())("expectation", countExpectation);
- return result;
-}
-
-std::shared_ptr<TColumnEngineChanges> TIntervalsOptimizerPlanner::DoGetOptimizationTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const {
- if (auto result = SizeProblemBlobs.BuildMergeTask(limits, granule, locksManager)) {
- return result;
- }
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("skip", "no_small_portion_tasks");
- return nullptr;
- if (RangedSegments.empty()) {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "no_ranged_segments");
- return nullptr;
- }
- auto& topSegment = **RangedSegments.rbegin()->second.begin();
- auto& features = topSegment.GetFeatures();
- std::vector<std::shared_ptr<TPortionInfo>> portions = GetPortionsForIntervalStartedIn(topSegment.GetPosition(), features.GetPortionsCount());
-
- if (portions.size() <= 1) {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_skip")("features", features.DebugJson().GetStringRobust())("reason", "one_portion");
- return nullptr;
- }
-
- std::optional<TString> tierName;
- for (auto&& i : portions) {
- if (i->GetMeta().GetTierName() && (!tierName || *tierName < i->GetMeta().GetTierName())) {
- tierName = i->GetMeta().GetTierName();
- }
- if (locksManager->IsLocked(*i)) {
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_skip")("features", features.DebugJson().GetStringRobust())
- ("count", features.GetPortionsCount())("reason", "busy_portion")("portion_address", i->GetAddress().DebugString());
- return nullptr;
- }
- }
- AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule")("features", features.DebugJson().GetStringRobust())("count", features.GetPortionsCount());
-
- TSaverContext saverContext(StoragesManager);
- return std::make_shared<NCompaction::TGeneralCompactColumnEngineChanges>(limits.GetSplitSettings(), granule, portions, saverContext);
-}
-
-void TIntervalsOptimizerPlanner::RemovePortion(const std::shared_ptr<TPortionInfo>& info) {
- AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "remove_portion")("portion_id", info->GetPortion());
- auto itStart = Positions.find(info->IndexKeyStart());
- auto itFinish = Positions.find(info->IndexKeyEnd());
- Y_ABORT_UNLESS(itStart != Positions.end());
- Y_ABORT_UNLESS(itFinish != Positions.end());
- if (itStart == itFinish) {
- RemoveRanged(itStart->second);
- itStart->second.RemoveSummary(info);
- AddRanged(itStart->second);
- if (itStart->second.RemoveStart(info) || itStart->second.RemoveFinish(info)) {
- RemoveRanged(itStart->second);
- Positions.erase(itStart);
- }
- } else {
- for (auto it = itStart; it != itFinish; ++it) {
- RemoveRanged(it->second);
- it->second.RemoveSummary(info);
- AddRanged(it->second);
- }
- if (itStart->second.RemoveStart(info)) {
- RemoveRanged(itStart->second);
- Positions.erase(itStart);
- }
- if (itFinish->second.RemoveFinish(info)) {
- RemoveRanged(itFinish->second);
- Positions.erase(itFinish);
- }
- }
- AFL_VERIFY(RangedSegments.empty() == Positions.empty())("rs_size", RangedSegments.size())("p_size", Positions.size());
-}
-
-void TIntervalsOptimizerPlanner::AddPortion(const std::shared_ptr<TPortionInfo>& info) {
- AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "add_portion")("portion_id", info->GetPortion());
- auto itStart = Positions.find(info->IndexKeyStart());
- if (itStart == Positions.end()) {
- itStart = Positions.emplace(info->IndexKeyStart(), TBorderPositions(info->IndexKeyStart())).first;
- if (itStart != Positions.begin()) {
- auto itStartCopy = itStart;
- --itStartCopy;
- itStart->second.CopyFrom(itStartCopy->second);
- AddRanged(itStart->second);
- }
- }
- auto itEnd = Positions.find(info->IndexKeyEnd());
- if (itEnd == Positions.end()) {
- itEnd = Positions.emplace(info->IndexKeyEnd(), TBorderPositions(info->IndexKeyEnd())).first;
- Y_ABORT_UNLESS(itEnd != Positions.begin());
- auto itEndCopy = itEnd;
- --itEndCopy;
- itEnd->second.CopyFrom(itEndCopy->second);
- AddRanged(itEnd->second);
- itStart = Positions.find(info->IndexKeyStart());
- }
- Y_ABORT_UNLESS(itStart != Positions.end());
- Y_ABORT_UNLESS(itEnd != Positions.end());
- itStart->second.AddStart(info);
- itEnd->second.AddFinish(info);
- if (itStart != itEnd) {
- for (auto it = itStart; it != itEnd; ++it) {
- RemoveRanged(it->second);
- it->second.AddSummary(info);
- AFL_VERIFY(!!it->second.GetFeatures());
- AddRanged(it->second);
- }
- } else {
- RemoveRanged(itStart->second);
- itStart->second.AddSummary(info);
- AddRanged(itStart->second);
- }
- AFL_VERIFY(RangedSegments.empty() == Positions.empty())("rs_size", RangedSegments.size())("p_size", Positions.size());
-}
-
-void TIntervalsOptimizerPlanner::DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) {
- for (auto&& [_, i] : remove) {
- SizeProblemBlobs.RemovePortion(i);
- RemovePortion(i);
- }
- for (auto&& [_, i] : add) {
- SizeProblemBlobs.AddPortion(i);
- AddPortion(i);
- }
-}
-
-void TIntervalsOptimizerPlanner::RemoveRanged(const TBorderPositions& data) {
- if (!!data.GetFeatures()) {
- Counters->OnRemoveIntervalsCount(data.GetFeatures().GetPortionsCount(), data.GetFeatures().GetPortionsRawWeight(), data.GetFeatures().GetPortionsWeight());
- auto itFeatures = RangedSegments.find(data.GetFeatures());
- Y_ABORT_UNLESS(itFeatures->second.erase(&data));
- if (itFeatures->second.empty()) {
- RangedSegments.erase(itFeatures);
- }
- }
-}
-
-void TIntervalsOptimizerPlanner::AddRanged(const TBorderPositions& data) {
- if (!!data.GetFeatures()) {
- Counters->OnAddIntervalsCount(data.GetFeatures().GetPortionsCount(), data.GetFeatures().GetPortionsRawWeight(), data.GetFeatures().GetPortionsWeight());
- Y_ABORT_UNLESS(RangedSegments[data.GetFeatures()].emplace(&data).second);
- }
-}
-
-TIntervalsOptimizerPlanner::TIntervalsOptimizerPlanner(const ui64 pathId, const std::shared_ptr<IStoragesManager>& storagesManager)
- : TBase(pathId)
- , StoragesManager(storagesManager)
- , Counters(std::make_shared<TCounters>())
- , SizeProblemBlobs(Counters, storagesManager)
-{
-}
-
-TOptimizationPriority TIntervalsOptimizerPlanner::DoGetUsefulMetric() const {
- auto res = SizeProblemBlobs.GetWeight();
- if (!!res) {
- AFL_VERIFY(RangedSegments.size())("positions", Positions.size())("sizes", SizeProblemBlobs.DebugString());
- return *res;
- }
- if (RangedSegments.empty()) {
- return TOptimizationPriority::Zero();
- }
- auto& topSegment = **RangedSegments.rbegin()->second.begin();
- auto& topFeaturesTask = topSegment.GetFeatures();
- return TOptimizationPriority::Optimization(topFeaturesTask.GetUsefulMetric());
-}
-
-TString TIntervalsOptimizerPlanner::DoDebugString() const {
- NJson::TJsonValue result = NJson::JSON_MAP;
- auto& positions = result.InsertValue("positions", NJson::JSON_ARRAY);
- for (auto&& i : Positions) {
- positions.AppendValue(i.second.DebugJson());
- }
- return result.GetStringRobust();
-}
-
-void TIntervalsOptimizerPlanner::TBorderPositions::AddSummary(const std::shared_ptr<TPortionInfo>& info) {
- Features.Add(info);
-}
-
-void TIntervalsOptimizerPlanner::TBorderPositions::RemoveSummary(const std::shared_ptr<TPortionInfo>& info) {
- Features.Remove(info);
-}
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h
deleted file mode 100644
index efe3a3ec61e..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h
+++ /dev/null
@@ -1,256 +0,0 @@
-#pragma once
-#include "optimizer.h"
-#include "counters.h"
-#include "blob_size.h"
-#include <ydb/core/formats/arrow/replace_key.h>
-#include <ydb/library/accessor/accessor.h>
-#include <ydb/core/tx/columnshard/splitter/settings.h>
-#include <ydb/core/tx/columnshard/blobs_action/abstract/write.h>
-#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-class TCounters;
-
-class TSegmentPosition {
-private:
- std::shared_ptr<TPortionInfo> Portion;
- const NArrow::TReplaceKey& Position;
- const bool IsStartFlag;
- TSegmentPosition(const std::shared_ptr<TPortionInfo>& data, const bool start)
- : Portion(data)
- , Position(start ? Portion->IndexKeyStart() : Portion->IndexKeyEnd())
- , IsStartFlag(start)
- {
-
- }
-public:
-
- NJson::TJsonValue DebugJson() const {
- NJson::TJsonValue result = NJson::JSON_MAP;
- result.InsertValue("is_start", IsStartFlag);
- return result;
- }
-
- TString DebugString() const {
- return TStringBuilder() <<
- (IsStartFlag ? "ADD" : "REMOVE") << ":" <<
- Position.DebugString() << ";" <<
- Portion->DebugString() << ";"
- ;
- }
-
- std::shared_ptr<TPortionInfo> GetPortionPtr() const {
- return Portion;
- }
-
- const TPortionInfo& GetPortion() const {
- return *Portion;
- }
-
- const NArrow::TReplaceKey& GetPosition() const {
- return Position;
- }
-
- static TSegmentPosition Start(const std::shared_ptr<TPortionInfo>& data) {
- return TSegmentPosition(data, true);
- }
-
- static TSegmentPosition Finish(const std::shared_ptr<TPortionInfo>& data) {
- return TSegmentPosition(data, false);
- }
-
- bool operator<(const TSegmentPosition& item) const {
- return Portion->GetPortion() < item.Portion->GetPortion();
- }
-};
-
-class TIntervalFeatures {
-private:
- YDB_READONLY(i32, PortionsCount, 0);
- YDB_READONLY(i32, RecordsCount, 0);
- YDB_READONLY(i64, PortionsWeight, 0);
- YDB_READONLY(i64, PortionsRawWeight, 0);
- YDB_READONLY(i64, SmallPortionsWeight, 0);
- YDB_READONLY(i64, SmallPortionsCount, 0);
-public:
-
- NJson::TJsonValue DebugJson() const {
- NJson::TJsonValue result = NJson::JSON_MAP;
- result.InsertValue("p_count", PortionsCount);
- result.InsertValue("p_weight", PortionsWeight);
- result.InsertValue("p_raw_weight", PortionsRawWeight);
- result.InsertValue("sp_count", SmallPortionsCount);
- result.InsertValue("sp_weight", SmallPortionsWeight);
- result.InsertValue("r_count", RecordsCount);
- return result;
- }
-
- i64 GetUsefulMetric() const {
- if (PortionsCount == 1 || PortionsWeight == 0) {
- return 0;
- }
- return (i64)10000 * PortionsCount / (PortionsWeight * 1e-6);
- }
-
- double GetUsefulKff() const {
- if (PortionsCount == 0 || PortionsWeight == 0) {
- return Max<double>();
- }
- Y_ABORT_UNLESS(PortionsWeight);
- return 1.0 * GetUsefulMetric() / PortionsWeight;
- }
-
- void Add(const std::shared_ptr<TPortionInfo>& info) {
- AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "add_portion_in_summary")("portion_id", info->GetPortion())("count", GetPortionsCount())("this", (ui64)this);
- ++PortionsCount;
- const i64 portionBytes = info->BlobsBytes();
- PortionsWeight += portionBytes;
- PortionsRawWeight += info->RawBytesSum();
- RecordsCount += info->NumRows();
- if ((i64)portionBytes < TSplitSettings().GetMinBlobSize()) {
- ++SmallPortionsCount;
- SmallPortionsWeight += portionBytes;
- }
- }
-
- void Remove(const std::shared_ptr<TPortionInfo>& info) {
- AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "remove_portion_from_summary")("portion_id", info->GetPortion())("count", GetPortionsCount())("this", (ui64)this);
- Y_ABORT_UNLESS(--PortionsCount >= 0);
- const i64 portionBytes = info->BlobsBytes();
- PortionsWeight -= portionBytes;
- Y_ABORT_UNLESS(PortionsWeight >= 0);
- PortionsRawWeight -= info->RawBytesSum();
- Y_ABORT_UNLESS(PortionsRawWeight >= 0);
- RecordsCount -= info->NumRows();
- Y_ABORT_UNLESS(RecordsCount >= 0);
- if ((i64)portionBytes < TSplitSettings().GetMinBlobSize()) {
- Y_ABORT_UNLESS(--SmallPortionsCount >= 0);
- SmallPortionsWeight -= portionBytes;
- Y_ABORT_UNLESS(SmallPortionsWeight >= 0);
- }
- }
-
- bool operator!() const {
- return !PortionsCount;
- }
-
- bool operator<(const TIntervalFeatures& item) const {
- return GetUsefulMetric() < item.GetUsefulMetric();
- }
- bool IsEnoughWeight() const {
- return GetPortionsRawWeight() > TSplitSettings().GetMinBlobSize() * 10;
- }
-
-};
-
-class TIntervalsOptimizerPlanner: public IOptimizerPlanner {
-private:
- static ui64 LimitSmallBlobsMerge;
- static ui64 LimitSmallBlobDetect;
- std::shared_ptr<IStoragesManager> StoragesManager;
-
- std::shared_ptr<TCounters> Counters;
-
- using TBase = IOptimizerPlanner;
-
- class TPortionIntervalPoint {
- private:
- YDB_READONLY(ui64, PortionId, 0);
- YDB_READONLY(bool, IsStart, false);
- public:
- TPortionIntervalPoint(const ui64 portionId, const bool isStart)
- : PortionId(portionId)
- , IsStart(isStart)
- {
-
- }
-
- bool operator<(const TPortionIntervalPoint& item) const {
- return std::tie(PortionId, IsStart) < std::tie(item.PortionId, item.IsStart);
- }
- };
-
- class TBorderPositions {
- private:
- const NArrow::TReplaceKey Position;
- std::map<TPortionIntervalPoint, TSegmentPosition> Positions;
- YDB_READONLY_DEF(TIntervalFeatures, Features);
- public:
- TBorderPositions(const NArrow::TReplaceKey& position)
- : Position(position)
- {
-
- }
-
- const std::map<TPortionIntervalPoint, TSegmentPosition>& GetPositions() const {
- return Positions;
- }
-
- NJson::TJsonValue DebugJson() const {
- NJson::TJsonValue result = NJson::JSON_MAP;
- result.InsertValue("p", Position.DebugString());
- auto& segments = result.InsertValue("segments", NJson::JSON_ARRAY);
- for (auto&& i : Positions) {
- segments.AppendValue(i.second.DebugJson());
- }
- result.InsertValue("features", Features.DebugJson());
- return result;
- }
-
- void CopyFrom(const TBorderPositions& source) {
- Features = source.Features;
- }
-
- const NArrow::TReplaceKey& GetPosition() const {
- return Position;
- }
-
- void AddStart(const std::shared_ptr<TPortionInfo>& info) {
- Y_ABORT_UNLESS(Positions.emplace(TPortionIntervalPoint(info->GetPortion(), true), TSegmentPosition::Start(info)).second);
- }
- void AddFinish(const std::shared_ptr<TPortionInfo>& info) {
- Y_ABORT_UNLESS(Positions.emplace(TPortionIntervalPoint(info->GetPortion(), false), TSegmentPosition::Finish(info)).second);
- }
- bool RemoveStart(const std::shared_ptr<TPortionInfo>& info) {
- Y_ABORT_UNLESS(Positions.erase(TPortionIntervalPoint(info->GetPortion(), true)));
- return Positions.empty();
- }
- bool RemoveFinish(const std::shared_ptr<TPortionInfo>& info) {
- Y_ABORT_UNLESS(Positions.erase(TPortionIntervalPoint(info->GetPortion(), false)));
- return Positions.empty();
- }
- void AddSummary(const std::shared_ptr<TPortionInfo>& info);
- void RemoveSummary(const std::shared_ptr<TPortionInfo>& info);
- };
-
- std::map<TIntervalFeatures, std::set<const TBorderPositions*>> RangedSegments;
-
- using TPositions = std::map<NArrow::TReplaceKey, TBorderPositions>;
- TPositions Positions;
- TBlobsBySize SizeProblemBlobs;
-
- void RemovePortion(const std::shared_ptr<TPortionInfo>& info);
- void AddPortion(const std::shared_ptr<TPortionInfo>& info);
-
- void RemoveRanged(const TBorderPositions& data);
- void AddRanged(const TBorderPositions& data);
-
- bool RemoveSmallPortion(const std::shared_ptr<TPortionInfo>& info);
-
- bool AddSmallPortion(const std::shared_ptr<TPortionInfo>& info);
-
- std::vector<std::shared_ptr<TPortionInfo>> GetPortionsForIntervalStartedIn(const NArrow::TReplaceKey& keyStart, const ui32 countExpectation) const;
-
-protected:
- virtual void DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) override;
- virtual std::shared_ptr<TColumnEngineChanges> DoGetOptimizationTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const override;
-
- virtual TOptimizationPriority DoGetUsefulMetric() const override;
- virtual TString DoDebugString() const override;
-
-public:
- TIntervalsOptimizerPlanner(const ui64 pathId, const std::shared_ptr<IStoragesManager>& storagesManager);
-};
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h
index 8d998773978..613891609d4 100644
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h
+++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h
@@ -201,6 +201,27 @@ private:
}
public:
+ bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const {
+ for (auto&& f : Futures) {
+ for (auto&& p : f.second) {
+ if (dataLocksManager->IsLocked(*p.second)) {
+ return true;
+ }
+ }
+ }
+ for (auto&& i : PreActuals) {
+ if (dataLocksManager->IsLocked(*i.second)) {
+ return true;
+ }
+ }
+ for (auto&& i : Actuals) {
+ if (dataLocksManager->IsLocked(*i.second)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
bool Validate(const std::shared_ptr<TPortionInfo>& portion) const {
if (portion) {
AFL_VERIFY(!PreActuals.contains(portion->GetPortionId()));
@@ -633,6 +654,13 @@ public:
}
};
+ bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const {
+ if (MainPortion && dataLocksManager->IsLocked(*MainPortion)) {
+ return true;
+ }
+ return Others.IsLocked(dataLocksManager);
+ }
+
bool IsEmpty() const {
return !MainPortion && Others.IsEmpty();
}
@@ -928,6 +956,17 @@ public:
AddBucketToRating(LeftBucket);
}
+ bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const {
+ if (BucketsByWeight.empty()) {
+ return false;
+ }
+ if (BucketsByWeight.rbegin()->second.empty()) {
+ return false;
+ }
+ const TPortionsBucket* bucketForOptimization = *BucketsByWeight.rbegin()->second.begin();
+ return bucketForOptimization->IsLocked(dataLocksManager);
+ }
+
bool IsEmpty() const {
return Buckets.empty() && LeftBucket->IsEmpty();
}
@@ -967,27 +1006,26 @@ public:
AFL_VERIFY(BucketsByWeight.size());
if (!BucketsByWeight.rbegin()->first) {
return nullptr;
+ }
+ AFL_VERIFY(BucketsByWeight.rbegin()->second.size());
+ const TPortionsBucket* bucketForOptimization = *BucketsByWeight.rbegin()->second.begin();
+ if (bucketForOptimization == LeftBucket.get()) {
+ if (Buckets.size()) {
+ return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &Buckets.begin()->first, PrimaryKeysSchema, StoragesManager);
+ } else {
+ return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager);
+ }
} else {
- AFL_VERIFY(BucketsByWeight.rbegin()->second.size());
- const TPortionsBucket* bucketForOptimization = *BucketsByWeight.rbegin()->second.begin();
- if (bucketForOptimization == LeftBucket.get()) {
- if (Buckets.size()) {
- return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &Buckets.begin()->first, PrimaryKeysSchema, StoragesManager);
- } else {
- return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager);
- }
+ auto it = Buckets.find(bucketForOptimization->GetPortion()->IndexKeyStart());
+ AFL_VERIFY(it != Buckets.end());
+ ++it;
+ if (it != Buckets.end()) {
+ return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &it->first, PrimaryKeysSchema, StoragesManager);
} else {
- auto it = Buckets.find(bucketForOptimization->GetPortion()->IndexKeyStart());
- AFL_VERIFY(it != Buckets.end());
- ++it;
- if (it != Buckets.end()) {
- return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &it->first, PrimaryKeysSchema, StoragesManager);
- } else {
- return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager);
- }
+ return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager);
}
}
- }
+ }
void AddPortion(const std::shared_ptr<TPortionInfo>& portion, const TInstant now) {
if (portion->GetBlobBytes() < SmallPortionDetectSizeLimit) {
@@ -1036,6 +1074,10 @@ private:
TPortionBuckets Buckets;
const std::shared_ptr<IStoragesManager> StoragesManager;
protected:
+ virtual bool DoIsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const override {
+ return Buckets.IsLocked(dataLocksManager);
+ }
+
virtual void DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) override {
const TInstant now = TInstant::Now();
for (auto&& [_, i] : remove) {
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp
deleted file mode 100644
index 6004ce09169..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-#include "counters.h"
-
-namespace NKikimr::NOlap::NStorageOptimizer::NLevels {
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h
deleted file mode 100644
index c0c3e2ca3d2..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h
+++ /dev/null
@@ -1,107 +0,0 @@
-#pragma once
-#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h>
-#include <ydb/core/formats/arrow/replace_key.h>
-#include <ydb/library/accessor/accessor.h>
-#include <ydb/core/tx/columnshard/splitter/settings.h>
-#include <ydb/core/tx/columnshard/counters/engine_logs.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer::NLevels {
-
-class TGlobalCounters: public NColumnShard::TCommonCountersOwner {
-private:
- using TBase = NColumnShard::TCommonCountersOwner;
- NMonitoring::TDynamicCounters::TCounterPtr SmallPortionsCount;
- std::shared_ptr<NColumnShard::TValueAggregationAgent> SmallPortionsCountByGranule;
-
- std::shared_ptr<NColumnShard::TValueAggregationAgent> CriticalRecordsCount;
- std::shared_ptr<NColumnShard::TValueAggregationAgent> NormalRecordsCount;
-
- std::shared_ptr<NColumnShard::TValueAggregationAgent> OldestCriticalActuality;
-public:
- TGlobalCounters()
- : TBase("LevelsStorageOptimizer")
- {
- SmallPortionsCount = TBase::GetValue("SmallPortions/Count");
- CriticalRecordsCount = TBase::GetValueAutoAggregations("Granule/CriticalRecord/Count");
- NormalRecordsCount = TBase::GetValueAutoAggregations("Granule/NormalRecord/Count");
- OldestCriticalActuality = TBase::GetValueAutoAggregations("Granule/ActualityMs");
- SmallPortionsCountByGranule = TBase::GetValueAutoAggregations("Granule/SmallPortions/Count");
- }
-
- static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildOldestCriticalActualityAggregation() {
- return Singleton<TGlobalCounters>()->OldestCriticalActuality->GetClient();
- }
-
- static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildClientSmallPortionsAggregation() {
- return Singleton<TGlobalCounters>()->SmallPortionsCountByGranule->GetClient();
- }
-
- static std::shared_ptr<NColumnShard::TValueGuard> BuildSmallPortionsGuard() {
- return std::make_shared<NColumnShard::TValueGuard>(Singleton<TGlobalCounters>()->SmallPortionsCount);
- }
-
- static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildCriticalRecordsCountAggregation() {
- return Singleton<TGlobalCounters>()->CriticalRecordsCount->GetClient();
- }
-
- static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildNormalRecordsCountAggregation() {
- return Singleton<TGlobalCounters>()->NormalRecordsCount->GetClient();
- }
-
-};
-
-class TCounters {
-private:
- std::shared_ptr<NColumnShard::TValueAggregationClient> CriticalRecordsCount;
- std::shared_ptr<NColumnShard::TValueAggregationClient> NormalRecordsCount;
-
- std::shared_ptr<NColumnShard::TValueAggregationClient> OldestCriticalActuality;
-
- std::shared_ptr<NColumnShard::TValueGuard> SmallPortionsCount;
- std::shared_ptr<NColumnShard::TValueAggregationClient> SmallPortionsByGranule;
-public:
- i64 GetSmallCounts() const {
- return SmallPortionsByGranule->GetValueSimple();
- }
-
- TCounters() {
- CriticalRecordsCount = TGlobalCounters::BuildCriticalRecordsCountAggregation();
- NormalRecordsCount = TGlobalCounters::BuildNormalRecordsCountAggregation();
- SmallPortionsCount = TGlobalCounters::BuildSmallPortionsGuard();
- SmallPortionsByGranule = TGlobalCounters::BuildClientSmallPortionsAggregation();
- OldestCriticalActuality = TGlobalCounters::BuildOldestCriticalActualityAggregation();
- }
-
- void OnMinProblemSnapshot(const TDuration d) {
- OldestCriticalActuality->SetValue(d.MilliSeconds(), TInstant::Now() + TDuration::Seconds(10));
- }
-
- void OnAddCriticalCount(const ui32 count) {
- CriticalRecordsCount->Add(count);
- }
-
- void OnAddNormalCount(const ui32 count) {
- NormalRecordsCount->Add(count);
- }
-
- void OnRemoveCriticalCount(const ui32 count) {
- CriticalRecordsCount->Remove(count);
- }
-
- void OnRemoveNormalCount(const ui32 count) {
- NormalRecordsCount->Remove(count);
- }
-
- void OnAddSmallPortion() {
- SmallPortionsCount->Add(1);
- SmallPortionsByGranule->Add(1);
- }
-
- void OnRemoveSmallPortion() {
- SmallPortionsCount->Sub(1);
- SmallPortionsByGranule->Remove(1);
- }
-
-};
-
-}
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp
deleted file mode 100644
index c78ef905041..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp
+++ /dev/null
@@ -1,5 +0,0 @@
-#include "optimizer.h"
-
-namespace NKikimr::NOlap::NStorageOptimizer {
-
-}
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h
deleted file mode 100644
index 67d7e1ae840..00000000000
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h
+++ /dev/null
@@ -1,524 +0,0 @@
-#pragma once
-#include "counters.h"
-
-#include <ydb/core/tx/columnshard/data_locks/manager/manager.h>
-#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h>
-#include <ydb/core/tx/columnshard/engines/changes/abstract/abstract.h>
-#include <ydb/core/tx/columnshard/engines/portions/portion_info.h>
-#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h>
-#include <ydb/core/tx/columnshard/engines/changes/general_compaction.h>
-#include <ydb/library/accessor/accessor.h>
-
-#include <util/generic/hash.h>
-#include <util/system/types.h>
-#include <util/generic/hash_set.h>
-
-namespace NKikimr::NOlap::NStorageOptimizer::NLevels {
-
-class TLevelInfo {
-private:
- THashMap<ui64, i64> Counters;
- YDB_READONLY(i64, CriticalWeight, 0);
- YDB_READONLY(i64, NormalizedWeight, 0);
- THashSet<ui64> PortionIds;
- std::shared_ptr<TCounters> Signals;
-public:
- TLevelInfo(std::shared_ptr<TCounters> counters)
- : Signals(counters)
- {
-
- }
-
- void AddPortion(const std::shared_ptr<TPortionInfo>& p, const ui32 refCount) {
- if (p->GetBlobBytes() < (1 << 20)) {
- Signals->OnAddSmallPortion();
- }
- auto it = Counters.find(p->GetPortion());
- i64 refCountPred = 0;
- if (it == Counters.end()) {
- it = Counters.emplace(p->GetPortion(), refCount).first;
- } else {
- refCountPred = it->second;
- it->second += refCount;
- }
- if (it->second == 1 && refCountPred == 0) {
- NormalizedWeight += p->NumRows();
- Signals->OnAddNormalCount(p->NumRows());
- } else if (it->second >= 2 && refCountPred == 1) {
- CriticalWeight += p->NumRows();
- Signals->OnAddCriticalCount(p->NumRows());
-
- NormalizedWeight -= p->NumRows();
- Y_ABORT_UNLESS(NormalizedWeight >= 0);
-
- Signals->OnRemoveNormalCount(p->NumRows());
- } else if (it->second >= 2 && refCountPred == 0) {
- CriticalWeight += p->NumRows();
- Signals->OnAddCriticalCount(p->NumRows());
- } else if (it->second >= 2 && refCountPred >= 2) {
- } else {
- Y_ABORT_UNLESS(false);
- }
- }
-
- void RemovePortion(const std::shared_ptr<TPortionInfo>& p, const ui32 refCount) {
- if (p->GetBlobBytes() < (1 << 20)) {
- Signals->OnRemoveSmallPortion();
- }
- auto it = Counters.find(p->GetPortion());
- Y_ABORT_UNLESS(it != Counters.end());
- const i64 refCountPred = it->second;
- it->second -= refCount;
- Y_ABORT_UNLESS(it->second >= 0);
- if (it->second >= 2) {
- } else if (it->second == 1) {
- Y_ABORT_UNLESS(refCountPred >= 2);
- CriticalWeight -= p->NumRows();
- Y_ABORT_UNLESS(CriticalWeight >= 0);
- Signals->OnRemoveCriticalCount(p->NumRows());
- Y_ABORT_UNLESS(CriticalWeight >= 0);
- NormalizedWeight += p->NumRows();
- Signals->OnAddNormalCount(p->NumRows());
- } else if (it->second == 0) {
- if (refCountPred >= 2) {
- Y_ABORT_UNLESS(refCountPred >= 2);
- CriticalWeight -= p->NumRows();
- Y_ABORT_UNLESS(CriticalWeight >= 0);
- Signals->OnRemoveCriticalCount(p->NumRows());
- } else if (refCountPred == 1) {
- NormalizedWeight -= p->NumRows();
- Y_ABORT_UNLESS(NormalizedWeight >= 0);
- Signals->OnRemoveNormalCount(p->NumRows());
- } else {
- Y_ABORT_UNLESS(false);
- }
- Counters.erase(it);
- }
- }
-
-};
-
-class TBorderPoint {
-public:
- using TBorderPortions = THashMap<ui64, std::shared_ptr<TPortionInfo>>;
-private:
- THashMap<ui64, ui32> MiddleWeight;
- YDB_READONLY_DEF(TBorderPortions, StartPortions);
- YDB_READONLY_DEF(TBorderPortions, MiddlePortions);
- YDB_READONLY_DEF(TBorderPortions, FinishPortions);
- std::shared_ptr<TLevelInfo> LevelInfo;
-public:
- void InitInternalPoint(const TBorderPoint& predPoint) {
- Y_ABORT_UNLESS(predPoint.MiddleWeight.size() == predPoint.MiddlePortions.size());
- for (auto&& i : predPoint.MiddlePortions) {
- auto it = predPoint.MiddleWeight.find(i.first);
- if (it->second != 2) {
- AddMiddle(i.second, 1);
- }
- }
- }
-
- std::shared_ptr<TPortionInfo> GetOnlyPortion() const {
- Y_ABORT_UNLESS(MiddlePortions.size() == 1);
- Y_ABORT_UNLESS(!IsCritical());
- return MiddlePortions.begin()->second;
- }
-
- bool IsSmall() const {
- if (!IsCritical() && MiddlePortions.size() == 1 && MiddlePortions.begin()->second->GetBlobBytes() < (1 << 20)) {
- return true;
- }
- return false;
- }
-
- bool IsCritical() const {
- if (StartPortions.size() && FinishPortions.size()) {
- return true;
- }
- if (MiddlePortions.size() > 1 || StartPortions.size() > 1 || FinishPortions.size() > 1) {
- return true;
- }
- return false;
- }
-
- TBorderPoint(const std::shared_ptr<TLevelInfo>& info)
- : LevelInfo(info) {
-
- }
-
- ~TBorderPoint() {
- for (auto&& i : MiddlePortions) {
- if (i.second->IndexKeyStart() == i.second->IndexKeyEnd()) {
- LevelInfo->RemovePortion(i.second, 2);
- } else {
- LevelInfo->RemovePortion(i.second, 1);
- }
- }
- }
-
- void AddStart(const std::shared_ptr<TPortionInfo>& p) {
- Y_ABORT_UNLESS(StartPortions.emplace(p->GetPortion(), p).second);
- }
- void RemoveStart(const std::shared_ptr<TPortionInfo>& p) {
- Y_ABORT_UNLESS(StartPortions.erase(p->GetPortion()));
- }
-
- void AddMiddle(const std::shared_ptr<TPortionInfo>& p, const ui32 portionCriticalWeight) {
- Y_ABORT_UNLESS(MiddleWeight.emplace(p->GetPortion(), portionCriticalWeight).second);
- Y_ABORT_UNLESS(MiddlePortions.emplace(p->GetPortion(), p).second);
- LevelInfo->AddPortion(p, portionCriticalWeight);
- }
- void RemoveMiddle(const std::shared_ptr<TPortionInfo>& p, const ui32 portionCriticalWeight) {
- Y_ABORT_UNLESS(MiddleWeight.erase(p->GetPortion()));
- Y_ABORT_UNLESS(MiddlePortions.erase(p->GetPortion()));
- LevelInfo->RemovePortion(p, portionCriticalWeight);
- }
-
- void AddFinish(const std::shared_ptr<TPortionInfo>& p) {
- Y_ABORT_UNLESS(FinishPortions.emplace(p->GetPortion(), p).second);
- }
- void RemoveFinish(const std::shared_ptr<TPortionInfo>& p) {
- Y_ABORT_UNLESS(FinishPortions.erase(p->GetPortion()));
- }
-
- bool IsEmpty() const {
- return StartPortions.empty() && FinishPortions.empty();
- }
-};
-
-class TPortionsPlacement {
-private:
- THashSet<ui64> PortionIds;
- std::map<NArrow::TReplaceKey, TBorderPoint> Borders;
- std::shared_ptr<TLevelInfo> LevelInfo;
-public:
- TPortionsPlacement(const std::shared_ptr<TLevelInfo>& levelInfo)
- : LevelInfo(levelInfo)
- {
-
- }
-
- class TPortionsScanner {
- private:
- THashMap<ui64, std::shared_ptr<TPortionInfo>> CurrentPortions;
- const std::shared_ptr<NDataLocks::TManager> DataLocksManager;
- public:
-
- TPortionsScanner(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager)
- : DataLocksManager(dataLocksManager)
- {
-
- }
-
- const THashMap<ui64, std::shared_ptr<TPortionInfo>>& GetCurrentPortions() const {
- return CurrentPortions;
- }
-
- bool AddBorderPoint(const TBorderPoint& p, bool& hasBusy) {
- hasBusy = false;
- for (auto&& [_, portionInfo] : p.GetStartPortions()) {
- if (DataLocksManager->IsLocked(*portionInfo)) {
- hasBusy = true;
- continue;
- }
- AFL_VERIFY(CurrentPortions.emplace(portionInfo->GetPortion(), portionInfo).second);
- }
-
- for (auto&& [_, portionInfo] : p.GetFinishPortions()) {
- if (DataLocksManager->IsLocked(*portionInfo)) {
- continue;
- }
- AFL_VERIFY(CurrentPortions.erase(portionInfo->GetPortion()));
- }
- return CurrentPortions.size();
- }
- };
-
- enum class EChainProblem {
- NoProblem,
- SmallChunks,
- MergeChunks
- };
-
- std::vector<std::vector<std::shared_ptr<TPortionInfo>>> GetPortionsToCompact(const ui64 sizeLimit, const std::shared_ptr<NDataLocks::TManager>& locksManager) const {
- std::vector<std::vector<std::shared_ptr<TPortionInfo>>> result;
- THashSet<ui64> readyPortionIds;
- ui64 resultSize = 0;
-
- TPortionsScanner buffer(locksManager);
- THashMap<ui64, std::shared_ptr<TPortionInfo>> portionsCurrentChain;
- ui64 chainSize = 0;
- EChainProblem problemType = EChainProblem::NoProblem;
- for (auto&& i : Borders) {
- bool hasBusy = false;
- if (!buffer.AddBorderPoint(i.second, hasBusy)) {
- if (hasBusy && problemType == EChainProblem::SmallChunks) {
- chainSize = 0;
- portionsCurrentChain.clear();
- problemType = EChainProblem::NoProblem;
- } else if (chainSize > (1 << 20)) {
- resultSize += chainSize;
- std::vector<std::shared_ptr<TPortionInfo>> chain;
- for (auto&& i : portionsCurrentChain) {
- chain.emplace_back(i.second);
- }
- result.emplace_back(chain);
- chainSize = 0;
- portionsCurrentChain.clear();
- problemType = EChainProblem::NoProblem;
- }
- } else {
- if (buffer.GetCurrentPortions().size() > 1) {
- problemType = EChainProblem::MergeChunks;
- } else if (buffer.GetCurrentPortions().begin()->second->GetBlobBytes() < (1 << 20) && problemType == EChainProblem::NoProblem) {
- problemType = EChainProblem::SmallChunks;
- }
- if (problemType != EChainProblem::NoProblem) {
- for (auto&& i : buffer.GetCurrentPortions()) {
- if (portionsCurrentChain.emplace(i.second->GetPortion(), i.second).second) {
- chainSize += i.second->GetBlobBytes();
- }
- }
- }
- }
- if (resultSize + chainSize > sizeLimit) {
- break;
- }
- }
- if (portionsCurrentChain.size() > 1) {
- std::vector<std::shared_ptr<TPortionInfo>> chain;
- for (auto&& i : portionsCurrentChain) {
- chain.emplace_back(i.second);
- }
- result.emplace_back(chain);
- }
-
- return result;
- }
-
- void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) {
- Y_ABORT_UNLESS(PortionIds.erase(portion->GetPortion()));
- auto itStart = Borders.find(portion->IndexKeyStart());
- AFL_VERIFY(itStart != Borders.end());
- auto itFinish = Borders.find(portion->IndexKeyEnd());
- AFL_VERIFY(itFinish != Borders.end());
-
- itStart->second.RemoveStart(portion);
- itFinish->second.RemoveFinish(portion);
- if (itStart != itFinish) {
- for (auto it = itStart; it != itFinish; ++it) {
- it->second.RemoveMiddle(portion, 1);
- }
- if (itFinish->second.IsEmpty()) {
- Y_ABORT_UNLESS(Borders.erase(portion->IndexKeyEnd()));
- }
- if (itStart->second.IsEmpty()) {
- Y_ABORT_UNLESS(Borders.erase(portion->IndexKeyStart()));
- }
- } else {
- itStart->second.RemoveMiddle(portion, 2);
- if (itStart->second.IsEmpty()) {
- Borders.erase(itStart);
- }
- }
- }
-
- void AddPortion(const std::shared_ptr<TPortionInfo>& portion) {
- Y_ABORT_UNLESS(PortionIds.emplace(portion->GetPortion()).second);
- auto itStartInfo = Borders.emplace(portion->IndexKeyStart(), TBorderPoint(LevelInfo));
- auto itStart = itStartInfo.first;
- if (itStartInfo.second && itStart != Borders.begin()) {
- auto itStartCopy = itStart;
- --itStartCopy;
- itStart->second.InitInternalPoint(itStartCopy->second);
- }
- auto itFinishInfo = Borders.emplace(portion->IndexKeyEnd(), TBorderPoint(LevelInfo));
- auto itFinish = itFinishInfo.first;
- if (itFinishInfo.second) {
- Y_ABORT_UNLESS(itFinish != Borders.begin());
- auto itFinishCopy = itFinish;
- --itFinishCopy;
- itFinish->second.InitInternalPoint(itFinishCopy->second);
- }
-
- itStart->second.AddStart(portion);
- itFinish->second.AddFinish(portion);
- if (itStart != itFinish) {
- for (auto it = itStart; it != itFinish; ++it) {
- it->second.AddMiddle(portion, 1);
- }
- } else {
- itStart->second.AddMiddle(portion, 2);
- }
- }
-};
-
-class TLevel {
-private:
- YDB_READONLY(TDuration, CriticalAge, TDuration::Zero());
- YDB_READONLY(ui64, CriticalSize, 0);
- std::shared_ptr<TLevelInfo> LevelInfo;
- TPortionsPlacement PortionsPlacement;
- std::shared_ptr<TLevel> NextLevel;
- std::map<NArrow::TReplaceKey, TBorderPoint> Borders;
- std::map<TSnapshot, THashMap<ui64, std::shared_ptr<TPortionInfo>>> PortionByAge;
- const ui64 PortionsSizeLimit = (ui64)250 * 1024 * 1024;
- TCompactionLimits CompactionLimits;
- THashSet<ui64> PortionIds;
- const std::shared_ptr<IStoragesManager> StoragesManager;
- std::shared_ptr<arrow::Schema> PrimaryKeysSchema;
-public:
- TLevel(const TDuration criticalAge, const ui64 criticalSize, std::shared_ptr<TLevel> nextLevel, const std::shared_ptr<IStoragesManager>& storagesManager, std::shared_ptr<TCounters> counters,
- const std::shared_ptr<arrow::Schema>& primaryKeysSchema)
- : CriticalAge(criticalAge)
- , CriticalSize(criticalSize)
- , LevelInfo(std::make_shared<TLevelInfo>(counters))
- , PortionsPlacement(LevelInfo)
- , NextLevel(nextLevel)
- , StoragesManager(storagesManager)
- , PrimaryKeysSchema(primaryKeysSchema)
- {
- CompactionLimits.GranuleSizeForOverloadPrevent = CriticalSize * 0.5;
- }
-
- ui64 GetWeight() const {
- return LevelInfo->GetCriticalWeight();
- }
-
- void ProvidePortionsNextLevel(const TInstant currentInstant) {
- if (!NextLevel) {
- return;
- }
- std::vector<std::shared_ptr<TPortionInfo>> portionsForProviding;
- for (auto&& i : PortionByAge) {
- if (TInstant::MilliSeconds(i.first.GetPlanStep()) + CriticalAge < currentInstant) {
- for (auto&& p : i.second) {
- portionsForProviding.emplace_back(p.second);
- }
- } else {
- break;
- }
- }
- for (auto&& i : portionsForProviding) {
- RemovePortion(i, currentInstant);
- NextLevel->AddPortion(i, currentInstant);
- }
- }
-
- void AddPortion(const std::shared_ptr<TPortionInfo>& portionInfo, const TInstant addInstant) {
- if (TInstant::MilliSeconds(portionInfo->RecordSnapshotMax().GetPlanStep()) + CriticalAge < addInstant) {
- Y_ABORT_UNLESS(!PortionIds.contains(portionInfo->GetPortion()));
- if (NextLevel) {
- return NextLevel->AddPortion(portionInfo, addInstant);
- }
- }
- PortionsPlacement.AddPortion(portionInfo);
- Y_ABORT_UNLESS(PortionByAge[portionInfo->RecordSnapshotMax()].emplace(portionInfo->GetPortion(), portionInfo).second);
- ProvidePortionsNextLevel(addInstant);
- }
-
- void RemovePortion(const std::shared_ptr<TPortionInfo>& portionInfo, const TInstant removeInstant) {
- PortionsPlacement.RemovePortion(portionInfo);
- {
- auto it = PortionByAge.find(portionInfo->RecordSnapshotMax());
- Y_ABORT_UNLESS(it != PortionByAge.end());
- Y_ABORT_UNLESS(it->second.erase(portionInfo->GetPortion()));
- if (it->second.empty()) {
- PortionByAge.erase(it);
- }
- }
- ProvidePortionsNextLevel(removeInstant);
- }
-
- std::shared_ptr<TColumnEngineChanges> BuildOptimizationTask(const TCompactionLimits& /*limits*/, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager, const TInstant /*currentInstant*/) const {
- std::vector<std::vector<std::shared_ptr<TPortionInfo>>> portionGroups = PortionsPlacement.GetPortionsToCompact(PortionsSizeLimit, locksManager);
- if (portionGroups.empty()) {
- AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "optimization_task_skipped");
- return nullptr;
- }
- std::vector<std::shared_ptr<TPortionInfo>> portions;
- std::vector<NIndexedReader::TSortableBatchPosition> positions;
- for (auto&& i : portionGroups) {
- portions.insert(portions.end(), i.begin(), i.end());
- std::optional<NIndexedReader::TSortableBatchPosition> position;
- for (auto&& p : i) {
- NIndexedReader::TSortableBatchPosition pos(p->IndexKeyEnd().ToBatch(PrimaryKeysSchema), 0, PrimaryKeysSchema->field_names(), {}, false);
- if (!position || position->Compare(pos) == std::partial_ordering::less) {
- position = pos;
- }
- }
- Y_ABORT_UNLESS(position);
- positions.emplace_back(*position);
- }
- TSaverContext saverContext(StoragesManager);
- auto result = std::make_shared<NCompaction::TGeneralCompactColumnEngineChanges>(CompactionLimits.GetSplitSettings(), granule, portions, saverContext);
- for (auto&& i : positions) {
- result->AddCheckPoint(i);
- }
- return result;
- }
-
-};
-
-class TLevelsOptimizerPlanner: public IOptimizerPlanner {
-private:
- using TBase = IOptimizerPlanner;
- std::shared_ptr<TLevel> L3;
- std::shared_ptr<TLevel> LMax;
- std::shared_ptr<TLevel> LStart;
- const std::shared_ptr<IStoragesManager> StoragesManager;
- std::shared_ptr<TCounters> Counters;
-protected:
- virtual std::vector<NIndexedReader::TSortableBatchPosition> GetBucketPositions() const override {
- return {};
- }
-
- virtual void DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) override {
- const TInstant currentInstant = TInstant::Now();
- for (auto&& [_, i] : remove) {
- if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") {
- continue;
- }
- if (!i->GetMeta().RecordSnapshotMax) {
- LMax->RemovePortion(i, currentInstant);
- } else {
- LStart->RemovePortion(i, currentInstant);
- }
- }
- for (auto&& [_, i] : add) {
- if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") {
- continue;
- }
- if (!i->GetMeta().RecordSnapshotMax) {
- LMax->AddPortion(i, currentInstant);
- } else {
- LStart->AddPortion(i, currentInstant);
- }
- }
- }
- virtual std::shared_ptr<TColumnEngineChanges> DoGetOptimizationTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const override {
- return LStart->BuildOptimizationTask(limits, granule, locksManager, TInstant::Now());
-
- }
- virtual TOptimizationPriority DoGetUsefulMetric() const override {
- return TOptimizationPriority::Critical(LStart->GetWeight());
- }
- virtual TString DoDebugString() const override {
- return "";
- }
- virtual void DoActualize(const TInstant /*currentInstant*/) override {
-
- }
-public:
- TLevelsOptimizerPlanner(const ui64 pathId, const std::shared_ptr<IStoragesManager>& storagesManager, const std::shared_ptr<arrow::Schema>& primaryKeysSchema)
- : TBase(pathId)
- , StoragesManager(storagesManager)
- , Counters(std::make_shared<TCounters>())
- {
- L3 = std::make_shared<TLevel>(TDuration::Seconds(120), 24 << 20, nullptr, StoragesManager, Counters, primaryKeysSchema);
- LMax = L3;
- LStart = L3;
- }
-};
-
-} // namespace NKikimr::NOlap
diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make
index e1362859aae..1d73127f26c 100644
--- a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make
+++ b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make
@@ -2,8 +2,6 @@ LIBRARY()
PEERDIR(
ydb/core/tx/columnshard/engines/storage/optimizer/abstract
- ydb/core/tx/columnshard/engines/storage/optimizer/intervals
- ydb/core/tx/columnshard/engines/storage/optimizer/levels
ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets
)
diff --git a/ydb/core/tx/columnshard/engines/storage/storage.cpp b/ydb/core/tx/columnshard/engines/storage/storage.cpp
index 722edeb5949..20c929dc70c 100644
--- a/ydb/core/tx/columnshard/engines/storage/storage.cpp
+++ b/ydb/core/tx/columnshard/engines/storage/storage.cpp
@@ -16,10 +16,10 @@ std::shared_ptr<NKikimr::NOlap::TGranuleMeta> TGranulesStorage::GetGranuleForCom
std::shared_ptr<TGranuleMeta> granule;
for (auto&& i : granules) {
i.second->ActualizeOptimizer(now);
- if (dataLocksManager->IsLocked(*i.second)) {
- continue;
- }
if (!priority || *priority < i.second->GetCompactionPriority()) {
+ if (i.second->IsLockedOptimizer(dataLocksManager)) {
+ continue;
+ }
priority = i.second->GetCompactionPriority();
granule = i.second;
}