diff options
| author | ivanmorozov333 <[email protected]> | 2024-03-12 07:57:23 +0300 |
|---|---|---|
| committer | GitHub <[email protected]> | 2024-03-12 07:57:23 +0300 |
| commit | 9e4e82efb3be63a2a9684441e4f66fac9e9dc2b0 (patch) | |
| tree | a0eb38d9dfe5fed9f73034fc3969beb90daf8a04 | |
| parent | 005091ff6cba7952d811036fdad0d6958a9ba474 (diff) | |
lighter lock - dont lock whole granule on background process (#2625)
15 files changed, 72 insertions, 1425 deletions
diff --git a/ydb/core/tx/columnshard/engines/storage/granule.h b/ydb/core/tx/columnshard/engines/storage/granule.h index 222a661091c..4ce5f6aec26 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule.h @@ -238,6 +238,10 @@ public: return OptimizerPlanner->GetUsefulMetric(); } + bool IsLockedOptimizer(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const { + return OptimizerPlanner->IsLocked(dataLocksManager); + } + void ActualizeOptimizer(const TInstant currentInstant) const { if (currentInstant - OptimizerPlanner->GetActualizationInstant() > TDuration::Seconds(1)) { OptimizerPlanner->Actualize(currentInstant); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h index e69d09e827c..31e91cd1bb4 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h @@ -67,7 +67,8 @@ protected: virtual NJson::TJsonValue DoSerializeToJsonVisual() const { return NJson::JSON_NULL; } - + virtual bool DoIsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const = 0; + public: using TFactory = NObjectFactory::TObjectFactory<IOptimizerPlanner, TString>; IOptimizerPlanner(const ui64 pathId) @@ -105,6 +106,7 @@ public: { } ~TModificationGuard() { + TMemoryProfileGuard mGuard("Optimizer"); Owner.ModifyPortions(AddPortions, RemovePortions); } }; @@ -119,6 +121,9 @@ public: } virtual std::vector<NIndexedReader::TSortableBatchPosition> GetBucketPositions() const = 0; + bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const { + return DoIsLocked(dataLocksManager); + } NJson::TJsonValue SerializeToJsonVisual() const { return DoSerializeToJsonVisual(); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp deleted file mode 100644 index e15eb551ca6..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "blob_size.h" -#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> -#include <ydb/core/tx/columnshard/engines/changes/general_compaction.h> -#include <ydb/core/tx/columnshard/data_locks/manager/manager.h> - -namespace NKikimr::NOlap::NStorageOptimizer { - -std::shared_ptr<NKikimr::NOlap::TColumnEngineChanges> TBlobsWithSizeLimit::BuildMergeTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const { - if (PortionsSize > (i64)SizeLimitToMerge || PortionsCount > CountLimitToMerge) { - i64 currentSum = 0; - std::vector<std::shared_ptr<TPortionInfo>> portions; - std::optional<TString> tierName; - for (auto&& i : Portions) { - for (auto&& c : i.second) { - if (locksManager->IsLocked(*c.second)) { - continue; - } - if (c.second->GetMeta().GetTierName() && (!tierName || *tierName < c.second->GetMeta().GetTierName())) { - tierName = c.second->GetMeta().GetTierName(); - } - currentSum += c.second->GetBlobBytes(); - portions.emplace_back(c.second); - if (currentSum > (i64)32 * 1024 * 1024) { - break; - } - } - if (currentSum > (i64)32 * 1024 * 1024) { - break; - } - } - if (currentSum > SizeLimitToMerge || PortionsCount > CountLimitToMerge) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("portions", portions.size())("current_sum", currentSum); - TSaverContext saverContext(StoragesManager); - return std::make_shared<NCompaction::TGeneralCompactColumnEngineChanges>(limits.GetSplitSettings(), granule, portions, saverContext); - } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("skip", "not_enough_data"); - } - } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_with_small")("event", "skip_by_condition"); - } - return nullptr; -} - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h deleted file mode 100644 index 4322d81bad4..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/blob_size.h +++ /dev/null @@ -1,145 +0,0 @@ -#pragma once -#include "counters.h" -#include <ydb/core/formats/arrow/replace_key.h> -#include <ydb/library/accessor/accessor.h> -#include <ydb/core/tx/columnshard/splitter/settings.h> -#include <ydb/core/tx/columnshard/blobs_action/abstract/write.h> -#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> - -namespace NKikimr::NOlap::NStorageOptimizer { - -class TBlobsWithSizeLimit { -private: - YDB_READONLY(ui64, SizeLimit, 0); - YDB_READONLY(i64, SizeLimitToMerge, (i64)2 * 1024 * 1024); - YDB_READONLY(i64, CountLimitToMerge, 8); - YDB_READONLY(i64, PortionsSize, 0); - YDB_READONLY(i64, PortionsCount, 0); - std::map<NArrow::TReplaceKey, std::map<ui64, std::shared_ptr<TPortionInfo>>> Portions; - std::shared_ptr<TCounters> Counters; - std::shared_ptr<IStoragesManager> StoragesManager; -public: - TString DebugString() const { - return TStringBuilder() - << "p_count=" << PortionsCount << ";" - << "p_count_by_key=" << Portions.size() << ";" - ; - } - - TBlobsWithSizeLimit(const ui64 limit, const std::shared_ptr<TCounters>& counters, const std::shared_ptr<IStoragesManager>& storagesManager) - : SizeLimit(limit) - , Counters(counters) - , StoragesManager(storagesManager) - { - - } - - std::shared_ptr<TColumnEngineChanges> BuildMergeTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const; - - void AddPortion(const std::shared_ptr<TPortionInfo>& portion) { - AFL_VERIFY(portion->BlobsBytes() < SizeLimit); - AFL_VERIFY(Portions[portion->IndexKeyStart()].emplace(portion->GetPortion(), portion).second); - PortionsSize += portion->BlobsBytes(); - ++PortionsCount; - Counters->OnAddSmallPortion(); - } - - void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) { - auto it = Portions.find(portion->IndexKeyStart()); - AFL_VERIFY(it != Portions.end()); - AFL_VERIFY(it->second.erase(portion->GetPortion())); - if (!it->second.size()) { - Portions.erase(it); - } - PortionsSize -= portion->BlobsBytes(); - AFL_VERIFY(PortionsSize >= 0); - --PortionsCount; - AFL_VERIFY(PortionsCount >= 0); - Counters->OnRemoveSmallPortion(); - } - - std::optional<TOptimizationPriority> GetWeight() const { - Y_ABORT_UNLESS(Counters->GetSmallCounts() == PortionsCount); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("portions_opt_count", PortionsCount)("counter", (ui64)Counters->SmallPortionsByGranule.get()); - if (PortionsSize > SizeLimitToMerge || PortionsCount > CountLimitToMerge) { - return TOptimizationPriority::Critical(PortionsCount); - } else { - return {}; - } - } -}; - -class TBlobsBySize { -private: - std::map<ui64, TBlobsWithSizeLimit> BlobsBySizeLimit; -public: - TString DebugString() const { - TStringBuilder sb; - sb << "("; - for (auto&& i : BlobsBySizeLimit) { - sb << "(" << i.first << ":" << i.second.DebugString() << ");"; - } - sb << ")"; - return sb; - } - - void AddPortion(const std::shared_ptr<TPortionInfo>& portion) { - auto it = BlobsBySizeLimit.upper_bound(portion->GetBlobBytes()); - if (it != BlobsBySizeLimit.end()) { - it->second.AddPortion(portion); - } - } - - void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) { - auto it = BlobsBySizeLimit.upper_bound(portion->GetBlobBytes()); - if (it != BlobsBySizeLimit.end()) { - it->second.RemovePortion(portion); - } - } - - std::optional<TOptimizationPriority> GetWeight() const { - std::optional<TOptimizationPriority> result; - for (auto&& i : BlobsBySizeLimit) { - auto w = i.second.GetWeight(); - if (!w) { - continue; - } - if (!result || *result < *w) { - result = w; - } - } - return result; - } - - const TBlobsWithSizeLimit* GetMaxWeightLimiter() const { - std::optional<TOptimizationPriority> resultWeight; - const TBlobsWithSizeLimit* result = nullptr; - for (auto&& i : BlobsBySizeLimit) { - auto w = i.second.GetWeight(); - if (!w) { - continue; - } - if (!resultWeight || *resultWeight < *w) { - resultWeight = w; - result = &i.second; - } - } - return result; - } - - std::shared_ptr<TColumnEngineChanges> BuildMergeTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const { - auto* limiter = GetMaxWeightLimiter(); - if (!limiter) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("fail", "limiter absent"); - return nullptr; - } - return limiter->BuildMergeTask(limits, granule, locksManager); - } - - TBlobsBySize(const std::shared_ptr<TCounters>& counters, const std::shared_ptr<IStoragesManager>& storagesManager) { -// BlobsBySizeLimit.emplace(512 * 1024, TBlobsWithSizeLimit(512 * 1024, counters, storagesManager)); - BlobsBySizeLimit.emplace(1024 * 1024, TBlobsWithSizeLimit(1024 * 1024, counters, storagesManager)); - } -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp deleted file mode 100644 index a8c1f6920c1..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "counters.h" - -namespace NKikimr::NOlap::NStorageOptimizer { - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h deleted file mode 100644 index e43f9acda67..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/counters.h +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once -#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h> -#include <ydb/core/formats/arrow/replace_key.h> -#include <ydb/library/accessor/accessor.h> -#include <ydb/core/tx/columnshard/splitter/settings.h> -#include <ydb/core/tx/columnshard/counters/engine_logs.h> - -namespace NKikimr::NOlap::NStorageOptimizer { - -class TGlobalCounters: public NColumnShard::TCommonCountersOwner { -private: - using TBase = NColumnShard::TCommonCountersOwner; - NMonitoring::TDynamicCounters::TCounterPtr SmallPortionsCount; - std::shared_ptr<NColumnShard::TIncrementalHistogram> HistogramOverlappedIntervalsCount; - std::shared_ptr<NColumnShard::TIncrementalHistogram> HistogramOverlappedIntervalsPackedSizeCount; - std::shared_ptr<NColumnShard::TIncrementalHistogram> HistogramOverlappedIntervalsRawSizeCount; - std::shared_ptr<NColumnShard::TValueAggregationAgent> SmallPortionsCountByGranule; -public: - TGlobalCounters() - : TBase("IntervalsStorageOptimizer") { - SmallPortionsCountByGranule = TBase::GetValueAutoAggregations("Granule/SmallPortions/Count"); - SmallPortionsCount = TBase::GetValue("SmallPortions/Count"); - - const std::set<i64> borders = {0, 1, 2, 4, 8, 16, 32, 64}; - HistogramOverlappedIntervalsCount = std::make_shared<NColumnShard::TIncrementalHistogram>("IntervalsStorageOptimizer", "OverlappedIntervals/Count", "", borders); - HistogramOverlappedIntervalsPackedSizeCount = std::make_shared<NColumnShard::TIncrementalHistogram>("IntervalsStorageOptimizer", "OverlappedIntervals/Size/Packed", "", borders); - HistogramOverlappedIntervalsRawSizeCount = std::make_shared<NColumnShard::TIncrementalHistogram>("IntervalsStorageOptimizer", "OverlappedIntervals/Size/Raw", "", borders); - } - - static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildClientSmallPortionsAggregation() { - return Singleton<TGlobalCounters>()->SmallPortionsCountByGranule->GetClient(); - } - - static std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> BuildGuardIntervalsOverlapping() { - return Singleton<TGlobalCounters>()->HistogramOverlappedIntervalsCount->BuildGuard(); - } - - static std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> BuildGuardIntervalsPackedSizeOverlapping() { - return Singleton<TGlobalCounters>()->HistogramOverlappedIntervalsPackedSizeCount->BuildGuard(); - } - - static std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> BuildGuardIntervalsRawSizeOverlapping() { - return Singleton<TGlobalCounters>()->HistogramOverlappedIntervalsRawSizeCount->BuildGuard(); - } - - static std::shared_ptr<NColumnShard::TValueGuard> BuildSmallPortionsGuard() { - return std::make_shared<NColumnShard::TValueGuard>(Singleton<TGlobalCounters>()->SmallPortionsCount); - } - -}; - -class TCounters { -private: - std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> IntervalsGuard; - std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> IntervalsPackedSizeGuard; - std::shared_ptr<NColumnShard::TIncrementalHistogram::TGuard> IntervalsRawSizeGuard; - std::shared_ptr<NColumnShard::TValueGuard> SmallPortionsCount; -public: - std::shared_ptr<NColumnShard::TValueAggregationClient> SmallPortionsByGranule; - i64 GetSmallCounts() const { - return SmallPortionsByGranule->GetValueSimple(); - } - - TCounters() { - IntervalsGuard = TGlobalCounters::BuildGuardIntervalsOverlapping(); - IntervalsPackedSizeGuard = TGlobalCounters::BuildGuardIntervalsPackedSizeOverlapping(); - IntervalsRawSizeGuard = TGlobalCounters::BuildGuardIntervalsRawSizeOverlapping(); - SmallPortionsCount = TGlobalCounters::BuildSmallPortionsGuard(); - SmallPortionsByGranule = TGlobalCounters::BuildClientSmallPortionsAggregation(); - } - - void OnRemoveIntervalsCount(const ui32 count, const ui64 rawSize, const ui64 packedSize) { - IntervalsGuard->Sub(count, 1); - IntervalsPackedSizeGuard->Sub(count, packedSize); - IntervalsRawSizeGuard->Sub(count, rawSize); - } - - void OnAddIntervalsCount(const ui32 count, const ui64 rawSize, const ui64 packedSize) { - IntervalsGuard->Add(count, 1); - IntervalsPackedSizeGuard->Add(count, packedSize); - IntervalsRawSizeGuard->Add(count, rawSize); - } - - void OnAddSmallPortion() { - SmallPortionsCount->Add(1); - SmallPortionsByGranule->Add(1); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("add_opt_count", SmallPortionsByGranule->GetValueSimple())("counter", (ui64)SmallPortionsByGranule.get()); - } - - void OnRemoveSmallPortion() { - SmallPortionsCount->Sub(1); - SmallPortionsByGranule->Remove(1); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("remove_opt_count", SmallPortionsByGranule->GetValueSimple())("counter", (ui64)SmallPortionsByGranule.get()); - } - -}; - -} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp deleted file mode 100644 index d9431021b4d..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.cpp +++ /dev/null @@ -1,213 +0,0 @@ -#include "optimizer.h" -#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> -#include <ydb/core/tx/columnshard/counters/common/owner.h> -#include <ydb/core/tx/columnshard/counters/engine_logs.h> -#include <ydb/core/tx/columnshard/data_locks/manager/manager.h> -#include <ydb/core/tx/columnshard/engines/changes/general_compaction.h> - -namespace NKikimr::NOlap::NStorageOptimizer { - -std::vector<std::shared_ptr<TPortionInfo>> TIntervalsOptimizerPlanner::GetPortionsForIntervalStartedIn(const NArrow::TReplaceKey& keyStart, const ui32 countExpectation) const { - std::vector<std::shared_ptr<TPortionInfo>> result; - auto it = Positions.find(keyStart); - AFL_VERIFY(it != Positions.end()); - THashSet<ui64> portionsCurrentlyClosed; - auto itReverse = make_reverse_iterator(it); - AFL_VERIFY(itReverse != Positions.rbegin()); - --itReverse; - for (; itReverse != Positions.rend(); ++itReverse) { - for (auto&& i : itReverse->second.GetPositions()) { - if (i.first.GetIsStart()) { - if (!portionsCurrentlyClosed.erase(i.first.GetPortionId())) { - result.emplace_back(i.second.GetPortionPtr()); - } - } else { - AFL_VERIFY(portionsCurrentlyClosed.emplace(i.first.GetPortionId()).second); - } - } - if (result.size() == countExpectation) { - return result; - } - } - AFL_VERIFY(false)("result.size()", result.size())("expectation", countExpectation); - return result; -} - -std::shared_ptr<TColumnEngineChanges> TIntervalsOptimizerPlanner::DoGetOptimizationTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const { - if (auto result = SizeProblemBlobs.BuildMergeTask(limits, granule, locksManager)) { - return result; - } - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("skip", "no_small_portion_tasks"); - return nullptr; - if (RangedSegments.empty()) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "no_ranged_segments"); - return nullptr; - } - auto& topSegment = **RangedSegments.rbegin()->second.begin(); - auto& features = topSegment.GetFeatures(); - std::vector<std::shared_ptr<TPortionInfo>> portions = GetPortionsForIntervalStartedIn(topSegment.GetPosition(), features.GetPortionsCount()); - - if (portions.size() <= 1) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_skip")("features", features.DebugJson().GetStringRobust())("reason", "one_portion"); - return nullptr; - } - - std::optional<TString> tierName; - for (auto&& i : portions) { - if (i->GetMeta().GetTierName() && (!tierName || *tierName < i->GetMeta().GetTierName())) { - tierName = i->GetMeta().GetTierName(); - } - if (locksManager->IsLocked(*i)) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule_skip")("features", features.DebugJson().GetStringRobust()) - ("count", features.GetPortionsCount())("reason", "busy_portion")("portion_address", i->GetAddress().DebugString()); - return nullptr; - } - } - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "take_granule")("features", features.DebugJson().GetStringRobust())("count", features.GetPortionsCount()); - - TSaverContext saverContext(StoragesManager); - return std::make_shared<NCompaction::TGeneralCompactColumnEngineChanges>(limits.GetSplitSettings(), granule, portions, saverContext); -} - -void TIntervalsOptimizerPlanner::RemovePortion(const std::shared_ptr<TPortionInfo>& info) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "remove_portion")("portion_id", info->GetPortion()); - auto itStart = Positions.find(info->IndexKeyStart()); - auto itFinish = Positions.find(info->IndexKeyEnd()); - Y_ABORT_UNLESS(itStart != Positions.end()); - Y_ABORT_UNLESS(itFinish != Positions.end()); - if (itStart == itFinish) { - RemoveRanged(itStart->second); - itStart->second.RemoveSummary(info); - AddRanged(itStart->second); - if (itStart->second.RemoveStart(info) || itStart->second.RemoveFinish(info)) { - RemoveRanged(itStart->second); - Positions.erase(itStart); - } - } else { - for (auto it = itStart; it != itFinish; ++it) { - RemoveRanged(it->second); - it->second.RemoveSummary(info); - AddRanged(it->second); - } - if (itStart->second.RemoveStart(info)) { - RemoveRanged(itStart->second); - Positions.erase(itStart); - } - if (itFinish->second.RemoveFinish(info)) { - RemoveRanged(itFinish->second); - Positions.erase(itFinish); - } - } - AFL_VERIFY(RangedSegments.empty() == Positions.empty())("rs_size", RangedSegments.size())("p_size", Positions.size()); -} - -void TIntervalsOptimizerPlanner::AddPortion(const std::shared_ptr<TPortionInfo>& info) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "add_portion")("portion_id", info->GetPortion()); - auto itStart = Positions.find(info->IndexKeyStart()); - if (itStart == Positions.end()) { - itStart = Positions.emplace(info->IndexKeyStart(), TBorderPositions(info->IndexKeyStart())).first; - if (itStart != Positions.begin()) { - auto itStartCopy = itStart; - --itStartCopy; - itStart->second.CopyFrom(itStartCopy->second); - AddRanged(itStart->second); - } - } - auto itEnd = Positions.find(info->IndexKeyEnd()); - if (itEnd == Positions.end()) { - itEnd = Positions.emplace(info->IndexKeyEnd(), TBorderPositions(info->IndexKeyEnd())).first; - Y_ABORT_UNLESS(itEnd != Positions.begin()); - auto itEndCopy = itEnd; - --itEndCopy; - itEnd->second.CopyFrom(itEndCopy->second); - AddRanged(itEnd->second); - itStart = Positions.find(info->IndexKeyStart()); - } - Y_ABORT_UNLESS(itStart != Positions.end()); - Y_ABORT_UNLESS(itEnd != Positions.end()); - itStart->second.AddStart(info); - itEnd->second.AddFinish(info); - if (itStart != itEnd) { - for (auto it = itStart; it != itEnd; ++it) { - RemoveRanged(it->second); - it->second.AddSummary(info); - AFL_VERIFY(!!it->second.GetFeatures()); - AddRanged(it->second); - } - } else { - RemoveRanged(itStart->second); - itStart->second.AddSummary(info); - AddRanged(itStart->second); - } - AFL_VERIFY(RangedSegments.empty() == Positions.empty())("rs_size", RangedSegments.size())("p_size", Positions.size()); -} - -void TIntervalsOptimizerPlanner::DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) { - for (auto&& [_, i] : remove) { - SizeProblemBlobs.RemovePortion(i); - RemovePortion(i); - } - for (auto&& [_, i] : add) { - SizeProblemBlobs.AddPortion(i); - AddPortion(i); - } -} - -void TIntervalsOptimizerPlanner::RemoveRanged(const TBorderPositions& data) { - if (!!data.GetFeatures()) { - Counters->OnRemoveIntervalsCount(data.GetFeatures().GetPortionsCount(), data.GetFeatures().GetPortionsRawWeight(), data.GetFeatures().GetPortionsWeight()); - auto itFeatures = RangedSegments.find(data.GetFeatures()); - Y_ABORT_UNLESS(itFeatures->second.erase(&data)); - if (itFeatures->second.empty()) { - RangedSegments.erase(itFeatures); - } - } -} - -void TIntervalsOptimizerPlanner::AddRanged(const TBorderPositions& data) { - if (!!data.GetFeatures()) { - Counters->OnAddIntervalsCount(data.GetFeatures().GetPortionsCount(), data.GetFeatures().GetPortionsRawWeight(), data.GetFeatures().GetPortionsWeight()); - Y_ABORT_UNLESS(RangedSegments[data.GetFeatures()].emplace(&data).second); - } -} - -TIntervalsOptimizerPlanner::TIntervalsOptimizerPlanner(const ui64 pathId, const std::shared_ptr<IStoragesManager>& storagesManager) - : TBase(pathId) - , StoragesManager(storagesManager) - , Counters(std::make_shared<TCounters>()) - , SizeProblemBlobs(Counters, storagesManager) -{ -} - -TOptimizationPriority TIntervalsOptimizerPlanner::DoGetUsefulMetric() const { - auto res = SizeProblemBlobs.GetWeight(); - if (!!res) { - AFL_VERIFY(RangedSegments.size())("positions", Positions.size())("sizes", SizeProblemBlobs.DebugString()); - return *res; - } - if (RangedSegments.empty()) { - return TOptimizationPriority::Zero(); - } - auto& topSegment = **RangedSegments.rbegin()->second.begin(); - auto& topFeaturesTask = topSegment.GetFeatures(); - return TOptimizationPriority::Optimization(topFeaturesTask.GetUsefulMetric()); -} - -TString TIntervalsOptimizerPlanner::DoDebugString() const { - NJson::TJsonValue result = NJson::JSON_MAP; - auto& positions = result.InsertValue("positions", NJson::JSON_ARRAY); - for (auto&& i : Positions) { - positions.AppendValue(i.second.DebugJson()); - } - return result.GetStringRobust(); -} - -void TIntervalsOptimizerPlanner::TBorderPositions::AddSummary(const std::shared_ptr<TPortionInfo>& info) { - Features.Add(info); -} - -void TIntervalsOptimizerPlanner::TBorderPositions::RemoveSummary(const std::shared_ptr<TPortionInfo>& info) { - Features.Remove(info); -} - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h deleted file mode 100644 index efe3a3ec61e..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/intervals/optimizer.h +++ /dev/null @@ -1,256 +0,0 @@ -#pragma once -#include "optimizer.h" -#include "counters.h" -#include "blob_size.h" -#include <ydb/core/formats/arrow/replace_key.h> -#include <ydb/library/accessor/accessor.h> -#include <ydb/core/tx/columnshard/splitter/settings.h> -#include <ydb/core/tx/columnshard/blobs_action/abstract/write.h> -#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> - -namespace NKikimr::NOlap::NStorageOptimizer { - -class TCounters; - -class TSegmentPosition { -private: - std::shared_ptr<TPortionInfo> Portion; - const NArrow::TReplaceKey& Position; - const bool IsStartFlag; - TSegmentPosition(const std::shared_ptr<TPortionInfo>& data, const bool start) - : Portion(data) - , Position(start ? Portion->IndexKeyStart() : Portion->IndexKeyEnd()) - , IsStartFlag(start) - { - - } -public: - - NJson::TJsonValue DebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue("is_start", IsStartFlag); - return result; - } - - TString DebugString() const { - return TStringBuilder() << - (IsStartFlag ? "ADD" : "REMOVE") << ":" << - Position.DebugString() << ";" << - Portion->DebugString() << ";" - ; - } - - std::shared_ptr<TPortionInfo> GetPortionPtr() const { - return Portion; - } - - const TPortionInfo& GetPortion() const { - return *Portion; - } - - const NArrow::TReplaceKey& GetPosition() const { - return Position; - } - - static TSegmentPosition Start(const std::shared_ptr<TPortionInfo>& data) { - return TSegmentPosition(data, true); - } - - static TSegmentPosition Finish(const std::shared_ptr<TPortionInfo>& data) { - return TSegmentPosition(data, false); - } - - bool operator<(const TSegmentPosition& item) const { - return Portion->GetPortion() < item.Portion->GetPortion(); - } -}; - -class TIntervalFeatures { -private: - YDB_READONLY(i32, PortionsCount, 0); - YDB_READONLY(i32, RecordsCount, 0); - YDB_READONLY(i64, PortionsWeight, 0); - YDB_READONLY(i64, PortionsRawWeight, 0); - YDB_READONLY(i64, SmallPortionsWeight, 0); - YDB_READONLY(i64, SmallPortionsCount, 0); -public: - - NJson::TJsonValue DebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue("p_count", PortionsCount); - result.InsertValue("p_weight", PortionsWeight); - result.InsertValue("p_raw_weight", PortionsRawWeight); - result.InsertValue("sp_count", SmallPortionsCount); - result.InsertValue("sp_weight", SmallPortionsWeight); - result.InsertValue("r_count", RecordsCount); - return result; - } - - i64 GetUsefulMetric() const { - if (PortionsCount == 1 || PortionsWeight == 0) { - return 0; - } - return (i64)10000 * PortionsCount / (PortionsWeight * 1e-6); - } - - double GetUsefulKff() const { - if (PortionsCount == 0 || PortionsWeight == 0) { - return Max<double>(); - } - Y_ABORT_UNLESS(PortionsWeight); - return 1.0 * GetUsefulMetric() / PortionsWeight; - } - - void Add(const std::shared_ptr<TPortionInfo>& info) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "add_portion_in_summary")("portion_id", info->GetPortion())("count", GetPortionsCount())("this", (ui64)this); - ++PortionsCount; - const i64 portionBytes = info->BlobsBytes(); - PortionsWeight += portionBytes; - PortionsRawWeight += info->RawBytesSum(); - RecordsCount += info->NumRows(); - if ((i64)portionBytes < TSplitSettings().GetMinBlobSize()) { - ++SmallPortionsCount; - SmallPortionsWeight += portionBytes; - } - } - - void Remove(const std::shared_ptr<TPortionInfo>& info) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "remove_portion_from_summary")("portion_id", info->GetPortion())("count", GetPortionsCount())("this", (ui64)this); - Y_ABORT_UNLESS(--PortionsCount >= 0); - const i64 portionBytes = info->BlobsBytes(); - PortionsWeight -= portionBytes; - Y_ABORT_UNLESS(PortionsWeight >= 0); - PortionsRawWeight -= info->RawBytesSum(); - Y_ABORT_UNLESS(PortionsRawWeight >= 0); - RecordsCount -= info->NumRows(); - Y_ABORT_UNLESS(RecordsCount >= 0); - if ((i64)portionBytes < TSplitSettings().GetMinBlobSize()) { - Y_ABORT_UNLESS(--SmallPortionsCount >= 0); - SmallPortionsWeight -= portionBytes; - Y_ABORT_UNLESS(SmallPortionsWeight >= 0); - } - } - - bool operator!() const { - return !PortionsCount; - } - - bool operator<(const TIntervalFeatures& item) const { - return GetUsefulMetric() < item.GetUsefulMetric(); - } - bool IsEnoughWeight() const { - return GetPortionsRawWeight() > TSplitSettings().GetMinBlobSize() * 10; - } - -}; - -class TIntervalsOptimizerPlanner: public IOptimizerPlanner { -private: - static ui64 LimitSmallBlobsMerge; - static ui64 LimitSmallBlobDetect; - std::shared_ptr<IStoragesManager> StoragesManager; - - std::shared_ptr<TCounters> Counters; - - using TBase = IOptimizerPlanner; - - class TPortionIntervalPoint { - private: - YDB_READONLY(ui64, PortionId, 0); - YDB_READONLY(bool, IsStart, false); - public: - TPortionIntervalPoint(const ui64 portionId, const bool isStart) - : PortionId(portionId) - , IsStart(isStart) - { - - } - - bool operator<(const TPortionIntervalPoint& item) const { - return std::tie(PortionId, IsStart) < std::tie(item.PortionId, item.IsStart); - } - }; - - class TBorderPositions { - private: - const NArrow::TReplaceKey Position; - std::map<TPortionIntervalPoint, TSegmentPosition> Positions; - YDB_READONLY_DEF(TIntervalFeatures, Features); - public: - TBorderPositions(const NArrow::TReplaceKey& position) - : Position(position) - { - - } - - const std::map<TPortionIntervalPoint, TSegmentPosition>& GetPositions() const { - return Positions; - } - - NJson::TJsonValue DebugJson() const { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue("p", Position.DebugString()); - auto& segments = result.InsertValue("segments", NJson::JSON_ARRAY); - for (auto&& i : Positions) { - segments.AppendValue(i.second.DebugJson()); - } - result.InsertValue("features", Features.DebugJson()); - return result; - } - - void CopyFrom(const TBorderPositions& source) { - Features = source.Features; - } - - const NArrow::TReplaceKey& GetPosition() const { - return Position; - } - - void AddStart(const std::shared_ptr<TPortionInfo>& info) { - Y_ABORT_UNLESS(Positions.emplace(TPortionIntervalPoint(info->GetPortion(), true), TSegmentPosition::Start(info)).second); - } - void AddFinish(const std::shared_ptr<TPortionInfo>& info) { - Y_ABORT_UNLESS(Positions.emplace(TPortionIntervalPoint(info->GetPortion(), false), TSegmentPosition::Finish(info)).second); - } - bool RemoveStart(const std::shared_ptr<TPortionInfo>& info) { - Y_ABORT_UNLESS(Positions.erase(TPortionIntervalPoint(info->GetPortion(), true))); - return Positions.empty(); - } - bool RemoveFinish(const std::shared_ptr<TPortionInfo>& info) { - Y_ABORT_UNLESS(Positions.erase(TPortionIntervalPoint(info->GetPortion(), false))); - return Positions.empty(); - } - void AddSummary(const std::shared_ptr<TPortionInfo>& info); - void RemoveSummary(const std::shared_ptr<TPortionInfo>& info); - }; - - std::map<TIntervalFeatures, std::set<const TBorderPositions*>> RangedSegments; - - using TPositions = std::map<NArrow::TReplaceKey, TBorderPositions>; - TPositions Positions; - TBlobsBySize SizeProblemBlobs; - - void RemovePortion(const std::shared_ptr<TPortionInfo>& info); - void AddPortion(const std::shared_ptr<TPortionInfo>& info); - - void RemoveRanged(const TBorderPositions& data); - void AddRanged(const TBorderPositions& data); - - bool RemoveSmallPortion(const std::shared_ptr<TPortionInfo>& info); - - bool AddSmallPortion(const std::shared_ptr<TPortionInfo>& info); - - std::vector<std::shared_ptr<TPortionInfo>> GetPortionsForIntervalStartedIn(const NArrow::TReplaceKey& keyStart, const ui32 countExpectation) const; - -protected: - virtual void DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) override; - virtual std::shared_ptr<TColumnEngineChanges> DoGetOptimizationTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const override; - - virtual TOptimizationPriority DoGetUsefulMetric() const override; - virtual TString DoDebugString() const override; - -public: - TIntervalsOptimizerPlanner(const ui64 pathId, const std::shared_ptr<IStoragesManager>& storagesManager); -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h index 8d998773978..613891609d4 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h @@ -201,6 +201,27 @@ private: } public: + bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const { + for (auto&& f : Futures) { + for (auto&& p : f.second) { + if (dataLocksManager->IsLocked(*p.second)) { + return true; + } + } + } + for (auto&& i : PreActuals) { + if (dataLocksManager->IsLocked(*i.second)) { + return true; + } + } + for (auto&& i : Actuals) { + if (dataLocksManager->IsLocked(*i.second)) { + return true; + } + } + return false; + } + bool Validate(const std::shared_ptr<TPortionInfo>& portion) const { if (portion) { AFL_VERIFY(!PreActuals.contains(portion->GetPortionId())); @@ -633,6 +654,13 @@ public: } }; + bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const { + if (MainPortion && dataLocksManager->IsLocked(*MainPortion)) { + return true; + } + return Others.IsLocked(dataLocksManager); + } + bool IsEmpty() const { return !MainPortion && Others.IsEmpty(); } @@ -928,6 +956,17 @@ public: AddBucketToRating(LeftBucket); } + bool IsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const { + if (BucketsByWeight.empty()) { + return false; + } + if (BucketsByWeight.rbegin()->second.empty()) { + return false; + } + const TPortionsBucket* bucketForOptimization = *BucketsByWeight.rbegin()->second.begin(); + return bucketForOptimization->IsLocked(dataLocksManager); + } + bool IsEmpty() const { return Buckets.empty() && LeftBucket->IsEmpty(); } @@ -967,27 +1006,26 @@ public: AFL_VERIFY(BucketsByWeight.size()); if (!BucketsByWeight.rbegin()->first) { return nullptr; + } + AFL_VERIFY(BucketsByWeight.rbegin()->second.size()); + const TPortionsBucket* bucketForOptimization = *BucketsByWeight.rbegin()->second.begin(); + if (bucketForOptimization == LeftBucket.get()) { + if (Buckets.size()) { + return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &Buckets.begin()->first, PrimaryKeysSchema, StoragesManager); + } else { + return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager); + } } else { - AFL_VERIFY(BucketsByWeight.rbegin()->second.size()); - const TPortionsBucket* bucketForOptimization = *BucketsByWeight.rbegin()->second.begin(); - if (bucketForOptimization == LeftBucket.get()) { - if (Buckets.size()) { - return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &Buckets.begin()->first, PrimaryKeysSchema, StoragesManager); - } else { - return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager); - } + auto it = Buckets.find(bucketForOptimization->GetPortion()->IndexKeyStart()); + AFL_VERIFY(it != Buckets.end()); + ++it; + if (it != Buckets.end()) { + return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &it->first, PrimaryKeysSchema, StoragesManager); } else { - auto it = Buckets.find(bucketForOptimization->GetPortion()->IndexKeyStart()); - AFL_VERIFY(it != Buckets.end()); - ++it; - if (it != Buckets.end()) { - return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, &it->first, PrimaryKeysSchema, StoragesManager); - } else { - return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager); - } + return bucketForOptimization->BuildOptimizationTask(limits, granule, locksManager, nullptr, PrimaryKeysSchema, StoragesManager); } } - } + } void AddPortion(const std::shared_ptr<TPortionInfo>& portion, const TInstant now) { if (portion->GetBlobBytes() < SmallPortionDetectSizeLimit) { @@ -1036,6 +1074,10 @@ private: TPortionBuckets Buckets; const std::shared_ptr<IStoragesManager> StoragesManager; protected: + virtual bool DoIsLocked(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) const override { + return Buckets.IsLocked(dataLocksManager); + } + virtual void DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) override { const TInstant now = TInstant::Now(); for (auto&& [_, i] : remove) { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp deleted file mode 100644 index 6004ce09169..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "counters.h" - -namespace NKikimr::NOlap::NStorageOptimizer::NLevels { - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h deleted file mode 100644 index c0c3e2ca3d2..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/counters.h +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once -#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h> -#include <ydb/core/formats/arrow/replace_key.h> -#include <ydb/library/accessor/accessor.h> -#include <ydb/core/tx/columnshard/splitter/settings.h> -#include <ydb/core/tx/columnshard/counters/engine_logs.h> - -namespace NKikimr::NOlap::NStorageOptimizer::NLevels { - -class TGlobalCounters: public NColumnShard::TCommonCountersOwner { -private: - using TBase = NColumnShard::TCommonCountersOwner; - NMonitoring::TDynamicCounters::TCounterPtr SmallPortionsCount; - std::shared_ptr<NColumnShard::TValueAggregationAgent> SmallPortionsCountByGranule; - - std::shared_ptr<NColumnShard::TValueAggregationAgent> CriticalRecordsCount; - std::shared_ptr<NColumnShard::TValueAggregationAgent> NormalRecordsCount; - - std::shared_ptr<NColumnShard::TValueAggregationAgent> OldestCriticalActuality; -public: - TGlobalCounters() - : TBase("LevelsStorageOptimizer") - { - SmallPortionsCount = TBase::GetValue("SmallPortions/Count"); - CriticalRecordsCount = TBase::GetValueAutoAggregations("Granule/CriticalRecord/Count"); - NormalRecordsCount = TBase::GetValueAutoAggregations("Granule/NormalRecord/Count"); - OldestCriticalActuality = TBase::GetValueAutoAggregations("Granule/ActualityMs"); - SmallPortionsCountByGranule = TBase::GetValueAutoAggregations("Granule/SmallPortions/Count"); - } - - static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildOldestCriticalActualityAggregation() { - return Singleton<TGlobalCounters>()->OldestCriticalActuality->GetClient(); - } - - static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildClientSmallPortionsAggregation() { - return Singleton<TGlobalCounters>()->SmallPortionsCountByGranule->GetClient(); - } - - static std::shared_ptr<NColumnShard::TValueGuard> BuildSmallPortionsGuard() { - return std::make_shared<NColumnShard::TValueGuard>(Singleton<TGlobalCounters>()->SmallPortionsCount); - } - - static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildCriticalRecordsCountAggregation() { - return Singleton<TGlobalCounters>()->CriticalRecordsCount->GetClient(); - } - - static std::shared_ptr<NColumnShard::TValueAggregationClient> BuildNormalRecordsCountAggregation() { - return Singleton<TGlobalCounters>()->NormalRecordsCount->GetClient(); - } - -}; - -class TCounters { -private: - std::shared_ptr<NColumnShard::TValueAggregationClient> CriticalRecordsCount; - std::shared_ptr<NColumnShard::TValueAggregationClient> NormalRecordsCount; - - std::shared_ptr<NColumnShard::TValueAggregationClient> OldestCriticalActuality; - - std::shared_ptr<NColumnShard::TValueGuard> SmallPortionsCount; - std::shared_ptr<NColumnShard::TValueAggregationClient> SmallPortionsByGranule; -public: - i64 GetSmallCounts() const { - return SmallPortionsByGranule->GetValueSimple(); - } - - TCounters() { - CriticalRecordsCount = TGlobalCounters::BuildCriticalRecordsCountAggregation(); - NormalRecordsCount = TGlobalCounters::BuildNormalRecordsCountAggregation(); - SmallPortionsCount = TGlobalCounters::BuildSmallPortionsGuard(); - SmallPortionsByGranule = TGlobalCounters::BuildClientSmallPortionsAggregation(); - OldestCriticalActuality = TGlobalCounters::BuildOldestCriticalActualityAggregation(); - } - - void OnMinProblemSnapshot(const TDuration d) { - OldestCriticalActuality->SetValue(d.MilliSeconds(), TInstant::Now() + TDuration::Seconds(10)); - } - - void OnAddCriticalCount(const ui32 count) { - CriticalRecordsCount->Add(count); - } - - void OnAddNormalCount(const ui32 count) { - NormalRecordsCount->Add(count); - } - - void OnRemoveCriticalCount(const ui32 count) { - CriticalRecordsCount->Remove(count); - } - - void OnRemoveNormalCount(const ui32 count) { - NormalRecordsCount->Remove(count); - } - - void OnAddSmallPortion() { - SmallPortionsCount->Add(1); - SmallPortionsByGranule->Add(1); - } - - void OnRemoveSmallPortion() { - SmallPortionsCount->Sub(1); - SmallPortionsByGranule->Remove(1); - } - -}; - -} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp deleted file mode 100644 index c78ef905041..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "optimizer.h" - -namespace NKikimr::NOlap::NStorageOptimizer { - -} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h deleted file mode 100644 index 67d7e1ae840..00000000000 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/levels/optimizer.h +++ /dev/null @@ -1,524 +0,0 @@ -#pragma once -#include "counters.h" - -#include <ydb/core/tx/columnshard/data_locks/manager/manager.h> -#include <ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h> -#include <ydb/core/tx/columnshard/engines/changes/abstract/abstract.h> -#include <ydb/core/tx/columnshard/engines/portions/portion_info.h> -#include <ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h> -#include <ydb/core/tx/columnshard/engines/changes/general_compaction.h> -#include <ydb/library/accessor/accessor.h> - -#include <util/generic/hash.h> -#include <util/system/types.h> -#include <util/generic/hash_set.h> - -namespace NKikimr::NOlap::NStorageOptimizer::NLevels { - -class TLevelInfo { -private: - THashMap<ui64, i64> Counters; - YDB_READONLY(i64, CriticalWeight, 0); - YDB_READONLY(i64, NormalizedWeight, 0); - THashSet<ui64> PortionIds; - std::shared_ptr<TCounters> Signals; -public: - TLevelInfo(std::shared_ptr<TCounters> counters) - : Signals(counters) - { - - } - - void AddPortion(const std::shared_ptr<TPortionInfo>& p, const ui32 refCount) { - if (p->GetBlobBytes() < (1 << 20)) { - Signals->OnAddSmallPortion(); - } - auto it = Counters.find(p->GetPortion()); - i64 refCountPred = 0; - if (it == Counters.end()) { - it = Counters.emplace(p->GetPortion(), refCount).first; - } else { - refCountPred = it->second; - it->second += refCount; - } - if (it->second == 1 && refCountPred == 0) { - NormalizedWeight += p->NumRows(); - Signals->OnAddNormalCount(p->NumRows()); - } else if (it->second >= 2 && refCountPred == 1) { - CriticalWeight += p->NumRows(); - Signals->OnAddCriticalCount(p->NumRows()); - - NormalizedWeight -= p->NumRows(); - Y_ABORT_UNLESS(NormalizedWeight >= 0); - - Signals->OnRemoveNormalCount(p->NumRows()); - } else if (it->second >= 2 && refCountPred == 0) { - CriticalWeight += p->NumRows(); - Signals->OnAddCriticalCount(p->NumRows()); - } else if (it->second >= 2 && refCountPred >= 2) { - } else { - Y_ABORT_UNLESS(false); - } - } - - void RemovePortion(const std::shared_ptr<TPortionInfo>& p, const ui32 refCount) { - if (p->GetBlobBytes() < (1 << 20)) { - Signals->OnRemoveSmallPortion(); - } - auto it = Counters.find(p->GetPortion()); - Y_ABORT_UNLESS(it != Counters.end()); - const i64 refCountPred = it->second; - it->second -= refCount; - Y_ABORT_UNLESS(it->second >= 0); - if (it->second >= 2) { - } else if (it->second == 1) { - Y_ABORT_UNLESS(refCountPred >= 2); - CriticalWeight -= p->NumRows(); - Y_ABORT_UNLESS(CriticalWeight >= 0); - Signals->OnRemoveCriticalCount(p->NumRows()); - Y_ABORT_UNLESS(CriticalWeight >= 0); - NormalizedWeight += p->NumRows(); - Signals->OnAddNormalCount(p->NumRows()); - } else if (it->second == 0) { - if (refCountPred >= 2) { - Y_ABORT_UNLESS(refCountPred >= 2); - CriticalWeight -= p->NumRows(); - Y_ABORT_UNLESS(CriticalWeight >= 0); - Signals->OnRemoveCriticalCount(p->NumRows()); - } else if (refCountPred == 1) { - NormalizedWeight -= p->NumRows(); - Y_ABORT_UNLESS(NormalizedWeight >= 0); - Signals->OnRemoveNormalCount(p->NumRows()); - } else { - Y_ABORT_UNLESS(false); - } - Counters.erase(it); - } - } - -}; - -class TBorderPoint { -public: - using TBorderPortions = THashMap<ui64, std::shared_ptr<TPortionInfo>>; -private: - THashMap<ui64, ui32> MiddleWeight; - YDB_READONLY_DEF(TBorderPortions, StartPortions); - YDB_READONLY_DEF(TBorderPortions, MiddlePortions); - YDB_READONLY_DEF(TBorderPortions, FinishPortions); - std::shared_ptr<TLevelInfo> LevelInfo; -public: - void InitInternalPoint(const TBorderPoint& predPoint) { - Y_ABORT_UNLESS(predPoint.MiddleWeight.size() == predPoint.MiddlePortions.size()); - for (auto&& i : predPoint.MiddlePortions) { - auto it = predPoint.MiddleWeight.find(i.first); - if (it->second != 2) { - AddMiddle(i.second, 1); - } - } - } - - std::shared_ptr<TPortionInfo> GetOnlyPortion() const { - Y_ABORT_UNLESS(MiddlePortions.size() == 1); - Y_ABORT_UNLESS(!IsCritical()); - return MiddlePortions.begin()->second; - } - - bool IsSmall() const { - if (!IsCritical() && MiddlePortions.size() == 1 && MiddlePortions.begin()->second->GetBlobBytes() < (1 << 20)) { - return true; - } - return false; - } - - bool IsCritical() const { - if (StartPortions.size() && FinishPortions.size()) { - return true; - } - if (MiddlePortions.size() > 1 || StartPortions.size() > 1 || FinishPortions.size() > 1) { - return true; - } - return false; - } - - TBorderPoint(const std::shared_ptr<TLevelInfo>& info) - : LevelInfo(info) { - - } - - ~TBorderPoint() { - for (auto&& i : MiddlePortions) { - if (i.second->IndexKeyStart() == i.second->IndexKeyEnd()) { - LevelInfo->RemovePortion(i.second, 2); - } else { - LevelInfo->RemovePortion(i.second, 1); - } - } - } - - void AddStart(const std::shared_ptr<TPortionInfo>& p) { - Y_ABORT_UNLESS(StartPortions.emplace(p->GetPortion(), p).second); - } - void RemoveStart(const std::shared_ptr<TPortionInfo>& p) { - Y_ABORT_UNLESS(StartPortions.erase(p->GetPortion())); - } - - void AddMiddle(const std::shared_ptr<TPortionInfo>& p, const ui32 portionCriticalWeight) { - Y_ABORT_UNLESS(MiddleWeight.emplace(p->GetPortion(), portionCriticalWeight).second); - Y_ABORT_UNLESS(MiddlePortions.emplace(p->GetPortion(), p).second); - LevelInfo->AddPortion(p, portionCriticalWeight); - } - void RemoveMiddle(const std::shared_ptr<TPortionInfo>& p, const ui32 portionCriticalWeight) { - Y_ABORT_UNLESS(MiddleWeight.erase(p->GetPortion())); - Y_ABORT_UNLESS(MiddlePortions.erase(p->GetPortion())); - LevelInfo->RemovePortion(p, portionCriticalWeight); - } - - void AddFinish(const std::shared_ptr<TPortionInfo>& p) { - Y_ABORT_UNLESS(FinishPortions.emplace(p->GetPortion(), p).second); - } - void RemoveFinish(const std::shared_ptr<TPortionInfo>& p) { - Y_ABORT_UNLESS(FinishPortions.erase(p->GetPortion())); - } - - bool IsEmpty() const { - return StartPortions.empty() && FinishPortions.empty(); - } -}; - -class TPortionsPlacement { -private: - THashSet<ui64> PortionIds; - std::map<NArrow::TReplaceKey, TBorderPoint> Borders; - std::shared_ptr<TLevelInfo> LevelInfo; -public: - TPortionsPlacement(const std::shared_ptr<TLevelInfo>& levelInfo) - : LevelInfo(levelInfo) - { - - } - - class TPortionsScanner { - private: - THashMap<ui64, std::shared_ptr<TPortionInfo>> CurrentPortions; - const std::shared_ptr<NDataLocks::TManager> DataLocksManager; - public: - - TPortionsScanner(const std::shared_ptr<NDataLocks::TManager>& dataLocksManager) - : DataLocksManager(dataLocksManager) - { - - } - - const THashMap<ui64, std::shared_ptr<TPortionInfo>>& GetCurrentPortions() const { - return CurrentPortions; - } - - bool AddBorderPoint(const TBorderPoint& p, bool& hasBusy) { - hasBusy = false; - for (auto&& [_, portionInfo] : p.GetStartPortions()) { - if (DataLocksManager->IsLocked(*portionInfo)) { - hasBusy = true; - continue; - } - AFL_VERIFY(CurrentPortions.emplace(portionInfo->GetPortion(), portionInfo).second); - } - - for (auto&& [_, portionInfo] : p.GetFinishPortions()) { - if (DataLocksManager->IsLocked(*portionInfo)) { - continue; - } - AFL_VERIFY(CurrentPortions.erase(portionInfo->GetPortion())); - } - return CurrentPortions.size(); - } - }; - - enum class EChainProblem { - NoProblem, - SmallChunks, - MergeChunks - }; - - std::vector<std::vector<std::shared_ptr<TPortionInfo>>> GetPortionsToCompact(const ui64 sizeLimit, const std::shared_ptr<NDataLocks::TManager>& locksManager) const { - std::vector<std::vector<std::shared_ptr<TPortionInfo>>> result; - THashSet<ui64> readyPortionIds; - ui64 resultSize = 0; - - TPortionsScanner buffer(locksManager); - THashMap<ui64, std::shared_ptr<TPortionInfo>> portionsCurrentChain; - ui64 chainSize = 0; - EChainProblem problemType = EChainProblem::NoProblem; - for (auto&& i : Borders) { - bool hasBusy = false; - if (!buffer.AddBorderPoint(i.second, hasBusy)) { - if (hasBusy && problemType == EChainProblem::SmallChunks) { - chainSize = 0; - portionsCurrentChain.clear(); - problemType = EChainProblem::NoProblem; - } else if (chainSize > (1 << 20)) { - resultSize += chainSize; - std::vector<std::shared_ptr<TPortionInfo>> chain; - for (auto&& i : portionsCurrentChain) { - chain.emplace_back(i.second); - } - result.emplace_back(chain); - chainSize = 0; - portionsCurrentChain.clear(); - problemType = EChainProblem::NoProblem; - } - } else { - if (buffer.GetCurrentPortions().size() > 1) { - problemType = EChainProblem::MergeChunks; - } else if (buffer.GetCurrentPortions().begin()->second->GetBlobBytes() < (1 << 20) && problemType == EChainProblem::NoProblem) { - problemType = EChainProblem::SmallChunks; - } - if (problemType != EChainProblem::NoProblem) { - for (auto&& i : buffer.GetCurrentPortions()) { - if (portionsCurrentChain.emplace(i.second->GetPortion(), i.second).second) { - chainSize += i.second->GetBlobBytes(); - } - } - } - } - if (resultSize + chainSize > sizeLimit) { - break; - } - } - if (portionsCurrentChain.size() > 1) { - std::vector<std::shared_ptr<TPortionInfo>> chain; - for (auto&& i : portionsCurrentChain) { - chain.emplace_back(i.second); - } - result.emplace_back(chain); - } - - return result; - } - - void RemovePortion(const std::shared_ptr<TPortionInfo>& portion) { - Y_ABORT_UNLESS(PortionIds.erase(portion->GetPortion())); - auto itStart = Borders.find(portion->IndexKeyStart()); - AFL_VERIFY(itStart != Borders.end()); - auto itFinish = Borders.find(portion->IndexKeyEnd()); - AFL_VERIFY(itFinish != Borders.end()); - - itStart->second.RemoveStart(portion); - itFinish->second.RemoveFinish(portion); - if (itStart != itFinish) { - for (auto it = itStart; it != itFinish; ++it) { - it->second.RemoveMiddle(portion, 1); - } - if (itFinish->second.IsEmpty()) { - Y_ABORT_UNLESS(Borders.erase(portion->IndexKeyEnd())); - } - if (itStart->second.IsEmpty()) { - Y_ABORT_UNLESS(Borders.erase(portion->IndexKeyStart())); - } - } else { - itStart->second.RemoveMiddle(portion, 2); - if (itStart->second.IsEmpty()) { - Borders.erase(itStart); - } - } - } - - void AddPortion(const std::shared_ptr<TPortionInfo>& portion) { - Y_ABORT_UNLESS(PortionIds.emplace(portion->GetPortion()).second); - auto itStartInfo = Borders.emplace(portion->IndexKeyStart(), TBorderPoint(LevelInfo)); - auto itStart = itStartInfo.first; - if (itStartInfo.second && itStart != Borders.begin()) { - auto itStartCopy = itStart; - --itStartCopy; - itStart->second.InitInternalPoint(itStartCopy->second); - } - auto itFinishInfo = Borders.emplace(portion->IndexKeyEnd(), TBorderPoint(LevelInfo)); - auto itFinish = itFinishInfo.first; - if (itFinishInfo.second) { - Y_ABORT_UNLESS(itFinish != Borders.begin()); - auto itFinishCopy = itFinish; - --itFinishCopy; - itFinish->second.InitInternalPoint(itFinishCopy->second); - } - - itStart->second.AddStart(portion); - itFinish->second.AddFinish(portion); - if (itStart != itFinish) { - for (auto it = itStart; it != itFinish; ++it) { - it->second.AddMiddle(portion, 1); - } - } else { - itStart->second.AddMiddle(portion, 2); - } - } -}; - -class TLevel { -private: - YDB_READONLY(TDuration, CriticalAge, TDuration::Zero()); - YDB_READONLY(ui64, CriticalSize, 0); - std::shared_ptr<TLevelInfo> LevelInfo; - TPortionsPlacement PortionsPlacement; - std::shared_ptr<TLevel> NextLevel; - std::map<NArrow::TReplaceKey, TBorderPoint> Borders; - std::map<TSnapshot, THashMap<ui64, std::shared_ptr<TPortionInfo>>> PortionByAge; - const ui64 PortionsSizeLimit = (ui64)250 * 1024 * 1024; - TCompactionLimits CompactionLimits; - THashSet<ui64> PortionIds; - const std::shared_ptr<IStoragesManager> StoragesManager; - std::shared_ptr<arrow::Schema> PrimaryKeysSchema; -public: - TLevel(const TDuration criticalAge, const ui64 criticalSize, std::shared_ptr<TLevel> nextLevel, const std::shared_ptr<IStoragesManager>& storagesManager, std::shared_ptr<TCounters> counters, - const std::shared_ptr<arrow::Schema>& primaryKeysSchema) - : CriticalAge(criticalAge) - , CriticalSize(criticalSize) - , LevelInfo(std::make_shared<TLevelInfo>(counters)) - , PortionsPlacement(LevelInfo) - , NextLevel(nextLevel) - , StoragesManager(storagesManager) - , PrimaryKeysSchema(primaryKeysSchema) - { - CompactionLimits.GranuleSizeForOverloadPrevent = CriticalSize * 0.5; - } - - ui64 GetWeight() const { - return LevelInfo->GetCriticalWeight(); - } - - void ProvidePortionsNextLevel(const TInstant currentInstant) { - if (!NextLevel) { - return; - } - std::vector<std::shared_ptr<TPortionInfo>> portionsForProviding; - for (auto&& i : PortionByAge) { - if (TInstant::MilliSeconds(i.first.GetPlanStep()) + CriticalAge < currentInstant) { - for (auto&& p : i.second) { - portionsForProviding.emplace_back(p.second); - } - } else { - break; - } - } - for (auto&& i : portionsForProviding) { - RemovePortion(i, currentInstant); - NextLevel->AddPortion(i, currentInstant); - } - } - - void AddPortion(const std::shared_ptr<TPortionInfo>& portionInfo, const TInstant addInstant) { - if (TInstant::MilliSeconds(portionInfo->RecordSnapshotMax().GetPlanStep()) + CriticalAge < addInstant) { - Y_ABORT_UNLESS(!PortionIds.contains(portionInfo->GetPortion())); - if (NextLevel) { - return NextLevel->AddPortion(portionInfo, addInstant); - } - } - PortionsPlacement.AddPortion(portionInfo); - Y_ABORT_UNLESS(PortionByAge[portionInfo->RecordSnapshotMax()].emplace(portionInfo->GetPortion(), portionInfo).second); - ProvidePortionsNextLevel(addInstant); - } - - void RemovePortion(const std::shared_ptr<TPortionInfo>& portionInfo, const TInstant removeInstant) { - PortionsPlacement.RemovePortion(portionInfo); - { - auto it = PortionByAge.find(portionInfo->RecordSnapshotMax()); - Y_ABORT_UNLESS(it != PortionByAge.end()); - Y_ABORT_UNLESS(it->second.erase(portionInfo->GetPortion())); - if (it->second.empty()) { - PortionByAge.erase(it); - } - } - ProvidePortionsNextLevel(removeInstant); - } - - std::shared_ptr<TColumnEngineChanges> BuildOptimizationTask(const TCompactionLimits& /*limits*/, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager, const TInstant /*currentInstant*/) const { - std::vector<std::vector<std::shared_ptr<TPortionInfo>>> portionGroups = PortionsPlacement.GetPortionsToCompact(PortionsSizeLimit, locksManager); - if (portionGroups.empty()) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "optimization_task_skipped"); - return nullptr; - } - std::vector<std::shared_ptr<TPortionInfo>> portions; - std::vector<NIndexedReader::TSortableBatchPosition> positions; - for (auto&& i : portionGroups) { - portions.insert(portions.end(), i.begin(), i.end()); - std::optional<NIndexedReader::TSortableBatchPosition> position; - for (auto&& p : i) { - NIndexedReader::TSortableBatchPosition pos(p->IndexKeyEnd().ToBatch(PrimaryKeysSchema), 0, PrimaryKeysSchema->field_names(), {}, false); - if (!position || position->Compare(pos) == std::partial_ordering::less) { - position = pos; - } - } - Y_ABORT_UNLESS(position); - positions.emplace_back(*position); - } - TSaverContext saverContext(StoragesManager); - auto result = std::make_shared<NCompaction::TGeneralCompactColumnEngineChanges>(CompactionLimits.GetSplitSettings(), granule, portions, saverContext); - for (auto&& i : positions) { - result->AddCheckPoint(i); - } - return result; - } - -}; - -class TLevelsOptimizerPlanner: public IOptimizerPlanner { -private: - using TBase = IOptimizerPlanner; - std::shared_ptr<TLevel> L3; - std::shared_ptr<TLevel> LMax; - std::shared_ptr<TLevel> LStart; - const std::shared_ptr<IStoragesManager> StoragesManager; - std::shared_ptr<TCounters> Counters; -protected: - virtual std::vector<NIndexedReader::TSortableBatchPosition> GetBucketPositions() const override { - return {}; - } - - virtual void DoModifyPortions(const THashMap<ui64, std::shared_ptr<TPortionInfo>>& add, const THashMap<ui64, std::shared_ptr<TPortionInfo>>& remove) override { - const TInstant currentInstant = TInstant::Now(); - for (auto&& [_, i] : remove) { - if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") { - continue; - } - if (!i->GetMeta().RecordSnapshotMax) { - LMax->RemovePortion(i, currentInstant); - } else { - LStart->RemovePortion(i, currentInstant); - } - } - for (auto&& [_, i] : add) { - if (i->GetMeta().GetTierName() != IStoragesManager::DefaultStorageId && i->GetMeta().GetTierName() != "") { - continue; - } - if (!i->GetMeta().RecordSnapshotMax) { - LMax->AddPortion(i, currentInstant); - } else { - LStart->AddPortion(i, currentInstant); - } - } - } - virtual std::shared_ptr<TColumnEngineChanges> DoGetOptimizationTask(const TCompactionLimits& limits, std::shared_ptr<TGranuleMeta> granule, const std::shared_ptr<NDataLocks::TManager>& locksManager) const override { - return LStart->BuildOptimizationTask(limits, granule, locksManager, TInstant::Now()); - - } - virtual TOptimizationPriority DoGetUsefulMetric() const override { - return TOptimizationPriority::Critical(LStart->GetWeight()); - } - virtual TString DoDebugString() const override { - return ""; - } - virtual void DoActualize(const TInstant /*currentInstant*/) override { - - } -public: - TLevelsOptimizerPlanner(const ui64 pathId, const std::shared_ptr<IStoragesManager>& storagesManager, const std::shared_ptr<arrow::Schema>& primaryKeysSchema) - : TBase(pathId) - , StoragesManager(storagesManager) - , Counters(std::make_shared<TCounters>()) - { - L3 = std::make_shared<TLevel>(TDuration::Seconds(120), 24 << 20, nullptr, StoragesManager, Counters, primaryKeysSchema); - LMax = L3; - LStart = L3; - } -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make index e1362859aae..1d73127f26c 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make @@ -2,8 +2,6 @@ LIBRARY() PEERDIR( ydb/core/tx/columnshard/engines/storage/optimizer/abstract - ydb/core/tx/columnshard/engines/storage/optimizer/intervals - ydb/core/tx/columnshard/engines/storage/optimizer/levels ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets ) diff --git a/ydb/core/tx/columnshard/engines/storage/storage.cpp b/ydb/core/tx/columnshard/engines/storage/storage.cpp index 722edeb5949..20c929dc70c 100644 --- a/ydb/core/tx/columnshard/engines/storage/storage.cpp +++ b/ydb/core/tx/columnshard/engines/storage/storage.cpp @@ -16,10 +16,10 @@ std::shared_ptr<NKikimr::NOlap::TGranuleMeta> TGranulesStorage::GetGranuleForCom std::shared_ptr<TGranuleMeta> granule; for (auto&& i : granules) { i.second->ActualizeOptimizer(now); - if (dataLocksManager->IsLocked(*i.second)) { - continue; - } if (!priority || *priority < i.second->GetCompactionPriority()) { + if (i.second->IsLockedOptimizer(dataLocksManager)) { + continue; + } priority = i.second->GetCompactionPriority(); granule = i.second; } |
