diff options
author | ivanmorozov333 <ivanmorozov@ydb.tech> | 2024-11-12 11:10:15 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-11-12 11:10:15 +0300 |
commit | cf0394ab6e19a9f38a9d083b49916ef41ff776bd (patch) | |
tree | 382f6644e201d89c166f1dc521bef893a8724234 | |
parent | a819f6783ef52e6e6b7b57d28c47e9029745bc75 (diff) | |
download | ydb-cf0394ab6e19a9f38a9d083b49916ef41ff776bd.tar.gz |
fix test and correct normalizer conditions (#11504)
5 files changed, 45 insertions, 10 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index f61aefa1b4..caf1ab0c13 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -18,7 +18,6 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteSmall ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.Select ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWrite ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertViaLegacyScripting-Streaming -ydb/core/tx/columnshard/engines/ut TColumnEngineTestLogs.IndexTtl ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_1_clean_with_restarts diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index c8e51e74fa..1869352394 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -515,15 +515,26 @@ std::shared_ptr<TSelectInfo> TColumnEngineForLogs::Select( return out; } -void TColumnEngineForLogs::OnTieringModified( - const std::shared_ptr<NColumnShard::TTiersManager>& manager, const NColumnShard::TTtl& ttl, const std::optional<ui64> pathId) { - if (!ActualizationStarted) { - for (auto&& i : GranulesStorage->GetTables()) { - i.second->StartActualizationIndex(); +bool TColumnEngineForLogs::StartActualization(const THashMap<ui64, TTiering>& specialPathEviction) { + if (ActualizationStarted) { + return false; + } + for (auto&& i : GranulesStorage->GetTables()) { + i.second->StartActualizationIndex(); + } + for (auto&& i : specialPathEviction) { + auto g = GetGranuleOptional(i.first); + if (g) { + g->RefreshTiering(i.second); } } - ActualizationStarted = true; + return true; +} + +void TColumnEngineForLogs::OnTieringModified( + const std::shared_ptr<NColumnShard::TTiersManager>& manager, const NColumnShard::TTtl& ttl, const std::optional<ui64> pathId) { + StartActualization({}); AFL_VERIFY(manager); THashMap<ui64, TTiering> tierings = manager->GetTiering(); ttl.AddTtls(tierings); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 56ff270e94..92f1a4ed5e 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -123,6 +123,7 @@ public: public: virtual std::shared_ptr<ITxReader> BuildLoader(const std::shared_ptr<IBlobGroupSelector>& dsGroupSelector) override; bool FinishLoading(); + bool StartActualization(const THashMap<ui64, TTiering>& specialPathEviction); virtual bool IsOverloadedByMetadata(const ui64 limit) const override { return limit < TGranulesStat::GetSumMetadataMemoryPortionsSize(); diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index 63a4cbbdea..4aab630892 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -432,7 +432,33 @@ bool Cleanup(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, u return result; } +namespace { +class TTestMetadataAccessorsSubscriber: public NOlap::IDataAccessorRequestsSubscriber { +private: + std::shared_ptr<IMetadataAccessorResultProcessor> Processor; + TColumnEngineForLogs& Engine; + + virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { + Processor->ApplyResult(std::move(result), Engine); + } + +public: + TTestMetadataAccessorsSubscriber(const std::shared_ptr<IMetadataAccessorResultProcessor>& processor, TColumnEngineForLogs& engine) + : Processor(processor) + , Engine(engine) { + } +}; + +} + bool Ttl(TColumnEngineForLogs& engine, TTestDbWrapper& db, const THashMap<ui64, NOlap::TTiering>& pathEviction, ui32 expectedToDrop) { + engine.StartActualization(pathEviction); + std::vector<NOlap::TCSMetadataRequest> requests = engine.CollectMetadataRequests(); + for (auto&& i : requests) { + i.GetRequest()->RegisterSubscriber(std::make_shared<TTestMetadataAccessorsSubscriber>(i.GetProcessor(), engine)); + engine.FetchDataAccessors(i.GetRequest()); + } + std::vector<std::shared_ptr<TTTLColumnEngineChanges>> vChanges = engine.StartTtl(pathEviction, EmptyDataLocksManager, 512 * 1024 * 1024); AFL_VERIFY(vChanges.size() == 1)("count", vChanges.size()); auto changes = vChanges.front(); diff --git a/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp index fac1467e98..a545e99ed7 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp @@ -145,9 +145,7 @@ TConclusion<std::vector<INormalizerTask::TPtr>> TNormalizer::DoInit( if (!ready) { return TConclusionStatus::Fail("Not ready"); } - if (!AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { - return std::vector<INormalizerTask::TPtr>(); - } + AFL_VERIFY(AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()); THashMap<ui64, TPortionLoadContext> portions0; THashSet<ui64> existPortions0; THashMap<ui64, std::map<TFullChunkAddress, TColumnChunkLoadContext>> columns0; |