aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorivanmorozov333 <ivanmorozov@ydb.tech>2024-11-12 11:10:15 +0300
committerGitHub <noreply@github.com>2024-11-12 11:10:15 +0300
commitcf0394ab6e19a9f38a9d083b49916ef41ff776bd (patch)
tree382f6644e201d89c166f1dc521bef893a8724234
parenta819f6783ef52e6e6b7b57d28c47e9029745bc75 (diff)
downloadydb-cf0394ab6e19a9f38a9d083b49916ef41ff776bd.tar.gz
fix test and correct normalizer conditions (#11504)
-rw-r--r--.github/config/muted_ya.txt1
-rw-r--r--ydb/core/tx/columnshard/engines/column_engine_logs.cpp23
-rw-r--r--ydb/core/tx/columnshard/engines/column_engine_logs.h1
-rw-r--r--ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp26
-rw-r--r--ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp4
5 files changed, 45 insertions, 10 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt
index f61aefa1b4..caf1ab0c13 100644
--- a/.github/config/muted_ya.txt
+++ b/.github/config/muted_ya.txt
@@ -18,7 +18,6 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.WriteSmall
ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.Select
ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertEvWrite
ydb/core/kqp/ut/data_integrity KqpDataIntegrityTrails.UpsertViaLegacyScripting-Streaming
-ydb/core/tx/columnshard/engines/ut TColumnEngineTestLogs.IndexTtl
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestAggregation
ydb/core/kqp/ut/olap KqpDecimalColumnShard.TestFilterCompare
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.BlobsSharingSplit1_1_clean_with_restarts
diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
index c8e51e74fa..1869352394 100644
--- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
+++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp
@@ -515,15 +515,26 @@ std::shared_ptr<TSelectInfo> TColumnEngineForLogs::Select(
return out;
}
-void TColumnEngineForLogs::OnTieringModified(
- const std::shared_ptr<NColumnShard::TTiersManager>& manager, const NColumnShard::TTtl& ttl, const std::optional<ui64> pathId) {
- if (!ActualizationStarted) {
- for (auto&& i : GranulesStorage->GetTables()) {
- i.second->StartActualizationIndex();
+bool TColumnEngineForLogs::StartActualization(const THashMap<ui64, TTiering>& specialPathEviction) {
+ if (ActualizationStarted) {
+ return false;
+ }
+ for (auto&& i : GranulesStorage->GetTables()) {
+ i.second->StartActualizationIndex();
+ }
+ for (auto&& i : specialPathEviction) {
+ auto g = GetGranuleOptional(i.first);
+ if (g) {
+ g->RefreshTiering(i.second);
}
}
-
ActualizationStarted = true;
+ return true;
+}
+
+void TColumnEngineForLogs::OnTieringModified(
+ const std::shared_ptr<NColumnShard::TTiersManager>& manager, const NColumnShard::TTtl& ttl, const std::optional<ui64> pathId) {
+ StartActualization({});
AFL_VERIFY(manager);
THashMap<ui64, TTiering> tierings = manager->GetTiering();
ttl.AddTtls(tierings);
diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h
index 56ff270e94..92f1a4ed5e 100644
--- a/ydb/core/tx/columnshard/engines/column_engine_logs.h
+++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h
@@ -123,6 +123,7 @@ public:
public:
virtual std::shared_ptr<ITxReader> BuildLoader(const std::shared_ptr<IBlobGroupSelector>& dsGroupSelector) override;
bool FinishLoading();
+ bool StartActualization(const THashMap<ui64, TTiering>& specialPathEviction);
virtual bool IsOverloadedByMetadata(const ui64 limit) const override {
return limit < TGranulesStat::GetSumMetadataMemoryPortionsSize();
diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp
index 63a4cbbdea..4aab630892 100644
--- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp
+++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp
@@ -432,7 +432,33 @@ bool Cleanup(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, u
return result;
}
+namespace {
+class TTestMetadataAccessorsSubscriber: public NOlap::IDataAccessorRequestsSubscriber {
+private:
+ std::shared_ptr<IMetadataAccessorResultProcessor> Processor;
+ TColumnEngineForLogs& Engine;
+
+ virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override {
+ Processor->ApplyResult(std::move(result), Engine);
+ }
+
+public:
+ TTestMetadataAccessorsSubscriber(const std::shared_ptr<IMetadataAccessorResultProcessor>& processor, TColumnEngineForLogs& engine)
+ : Processor(processor)
+ , Engine(engine) {
+ }
+};
+
+}
+
bool Ttl(TColumnEngineForLogs& engine, TTestDbWrapper& db, const THashMap<ui64, NOlap::TTiering>& pathEviction, ui32 expectedToDrop) {
+ engine.StartActualization(pathEviction);
+ std::vector<NOlap::TCSMetadataRequest> requests = engine.CollectMetadataRequests();
+ for (auto&& i : requests) {
+ i.GetRequest()->RegisterSubscriber(std::make_shared<TTestMetadataAccessorsSubscriber>(i.GetProcessor(), engine));
+ engine.FetchDataAccessors(i.GetRequest());
+ }
+
std::vector<std::shared_ptr<TTTLColumnEngineChanges>> vChanges = engine.StartTtl(pathEviction, EmptyDataLocksManager, 512 * 1024 * 1024);
AFL_VERIFY(vChanges.size() == 1)("count", vChanges.size());
auto changes = vChanges.front();
diff --git a/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp
index fac1467e98..a545e99ed7 100644
--- a/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp
+++ b/ydb/core/tx/columnshard/normalizer/portion/restore_v1_chunks.cpp
@@ -145,9 +145,7 @@ TConclusion<std::vector<INormalizerTask::TPtr>> TNormalizer::DoInit(
if (!ready) {
return TConclusionStatus::Fail("Not ready");
}
- if (!AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) {
- return std::vector<INormalizerTask::TPtr>();
- }
+ AFL_VERIFY(AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage());
THashMap<ui64, TPortionLoadContext> portions0;
THashSet<ui64> existPortions0;
THashMap<ui64, std::map<TFullChunkAddress, TColumnChunkLoadContext>> columns0;