diff options
author | snaury <[email protected]> | 2022-07-12 15:52:22 +0300 |
---|---|---|
committer | snaury <[email protected]> | 2022-07-12 15:52:22 +0300 |
commit | 75c170827c1cb55af373fa50878657734a06329f (patch) | |
tree | 4f148cd6625f6686b0506745113b57fe44cc1fb4 | |
parent | c240e4b5c407684b6dd8bf9328c1a0d88efcec8f (diff) |
Remove fast split support,
5 files changed, 0 insertions, 154 deletions
diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp index 5e5cdef9efd..446495fff7d 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp @@ -376,10 +376,6 @@ bool TTxStorePartitionStats::PersistSingleStats(TTransactionContext& txc, const } else if (table->GetPartitions().size() >= table->GetMaxPartitionsCount()) { // We cannot split as there are max partitions already return true; - } else if (table->CheckFastSplitForPartition(Self->SplitSettings, shardIdx, dataSize, rowCount)) { - dataSizeResolution = Max<ui64>(dataSize / 100, 100*1024); - rowCountResolution = Max<ui64>(rowCount / 100, 1000); - collectKeySample = true; } else if (table->CheckSplitByLoad(Self->SplitSettings, shardIdx, dataSize, rowCount)) { collectKeySample = true; } else { diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp index 0c5adea88cc..1001f22c761 100644 --- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp @@ -315,13 +315,6 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex splitReason = ESplitReason::SPLIT_BY_SIZE; } - if (splitReason == ESplitReason::NO_SPLIT && table->CheckFastSplitForPartition(Self->SplitSettings, shardIdx, dataSize, rowCount)) { - const TTableInfo* parentTable = Self->GetMainTableForIndex(tableId); - if (parentTable && table->GetPartitions().size() < parentTable->GetPartitions().size()) { - splitReason = ESplitReason::FAST_SPLIT_INDEX; - } - } - if (splitReason == ESplitReason::NO_SPLIT && table->CheckSplitByLoad(Self->SplitSettings, shardIdx, dataSize, rowCount)) { splitReason = ESplitReason::SPLIT_BY_LOAD; } diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index ee068bbfb28..5fcd1204e8a 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -1557,44 +1557,6 @@ bool TTableInfo::CheckCanMergePartitions(const TSplitSettings& splitSettings, return shardsToMerge.size() > 1; } -bool TTableInfo::CheckFastSplitForPartition(const TSplitSettings& splitSettings, TShardIdx shardIdx, ui64 dataSize, ui64 rowCount) const { - // Don't split/merge backup tables - if (IsBackup) - return false; - - // Ignore stats from unknown datashard (it could have been split) - if (!Stats.PartitionStats.contains(shardIdx)) - return false; - - if (!Shard2PartitionIdx.contains(shardIdx)) - return false; - - const ui64 MIN_ROWS_FOR_FAST_SPLIT = 1000; - ui64 sizeThreshold = splitSettings.FastSplitSizeThreshold; - ui64 rowCountThreshold = splitSettings.FastSplitRowCountThreshold; - float cpuUsageThreshold = 0.01 * splitSettings.FastSplitCpuPercentageThreshold; - - const auto& partitionConfig = PartitionConfig(); - - if (partitionConfig.GetPartitioningPolicy().HasFastSplitSettings()) { - const auto& settings = partitionConfig.GetPartitioningPolicy().GetFastSplitSettings(); - sizeThreshold = settings.GetSizeThreshold(); - rowCountThreshold = settings.GetRowCountThreshold(); - cpuUsageThreshold = 0.01 * settings.GetCpuPercentageThreshold(); - } - - const auto& stats = *Stats.PartitionStats.FindPtr(shardIdx); - if ((dataSize < sizeThreshold && rowCount < rowCountThreshold) || - rowCount < MIN_ROWS_FOR_FAST_SPLIT || - stats.InFlightTxCount == 0 || - stats.GetCurrentRawCpuUsage() < cpuUsageThreshold * 1000000) - { - return false; - } - - return true; -} - bool TTableInfo::CheckSplitByLoad(const TSplitSettings& splitSettings, TShardIdx shardIdx, ui64 dataSize, ui64 rowCount) const { // Don't split/merge backup tables if (IsBackup) diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index b2e6a23e925..010dd51c8f2 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -592,7 +592,6 @@ public: const TForceShardSplitSettings& forceShardSplitSettings, TShardIdx shardIdx, TVector<TShardIdx>& shardsToMerge) const; - bool CheckFastSplitForPartition(const TSplitSettings& splitSettings, TShardIdx shardIdx, ui64 dataSize, ui64 rowCount) const; bool CheckSplitByLoad(const TSplitSettings& splitSettings, TShardIdx shardIdx, ui64 dataSize, ui64 rowCount) const; bool IsSplitBySizeEnabled(const TForceShardSplitSettings& params) const { diff --git a/ydb/services/ydb/ydb_index_table_ut.cpp b/ydb/services/ydb/ydb_index_table_ut.cpp index e069859e53b..4459e001582 100644 --- a/ydb/services/ydb/ydb_index_table_ut.cpp +++ b/ydb/services/ydb/ydb_index_table_ut.cpp @@ -43,110 +43,6 @@ void CreateTestTableWithIndex(NYdb::NTable::TTableClient& client) { } Y_UNIT_TEST_SUITE(YdbIndexTable) { - Y_UNIT_TEST(FastSplitIndex) { - TKikimrWithGrpcAndRootSchema server; - - NYdb::TDriver driver(TDriverConfig().SetEndpoint(TStringBuilder() << "localhost:" << server.GetPort())); - NYdb::NTable::TTableClient client(driver); - NFlatTests::TFlatMsgBusClient oldClient(server.ServerSettings->Port); - - CreateTestTableWithIndex(client); - - size_t shardsBefore = oldClient.GetTablePartitions("/Root/Foo/TimestampIndex/indexImplTable").size(); - Cerr << "Index table has " << shardsBefore << " shards" << Endl; - UNIT_ASSERT_VALUES_EQUAL(shardsBefore, 1); - - NDataShard::gDbStatsReportInterval = TDuration::Seconds(0); - server.Server_->GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_NOTICE); - - // Set low CPU usage threshold for robustness - TAtomic unused; - server.Server_->GetRuntime()->GetAppData().Icb->SetValue("SchemeShard_FastSplitCpuPercentageThreshold", 1, unused); - server.Server_->GetRuntime()->GetAppData().Icb->SetValue("DataShardControls.CpuUsageReportThreshlodPercent", 1, unused); - server.Server_->GetRuntime()->GetAppData().Icb->SetValue("DataShardControls.CpuUsageReportIntervalSeconds", 3, unused); - - TString query = - "DECLARE $name_hash AS Uint32;\n" - "DECLARE $name AS Utf8;\n" - "DECLARE $version AS Uint32;\n" - "DECLARE $timestamp AS Int64;\n\n" - "UPSERT INTO `/Root/Foo` (NameHash, Name, Version, Timestamp) " - " VALUES ($name_hash, $name, $version, $timestamp);"; - - TAtomic enough = 0; - auto threadFunc = [&client, &query, &enough](TString namePrefix) { - auto sessionResult = client.CreateSession().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(sessionResult.GetStatus(), EStatus::SUCCESS); - auto session = sessionResult.GetSession(); - - for (int key = 0 ; key < 2000 && !AtomicGet(enough); ++key) { - TString name = namePrefix + ToString(key); - - auto paramsBuilder = client.GetParamsBuilder(); - auto params = paramsBuilder - .AddParam("$name_hash") - .Uint32(MurmurHash<ui32>(name.data(), name.size())) - .Build() - .AddParam("$name") - .Utf8(name) - .Build() - .AddParam("$version") - .Uint32(key%5) - .Build() - .AddParam("$timestamp") - .Int64(key%10) - .Build() - .Build(); - - auto result = session.ExecuteDataQuery( - query, - TTxControl::BeginTx().CommitTx(), std::move(params)).ExtractValueSync(); - - if (!result.IsSuccess() && result.GetStatus() != NYdb::EStatus::OVERLOADED) { - TString err = result.GetIssues().ToString(); - Cerr << result.GetStatus() << ": " << err << Endl; - } - UNIT_ASSERT_EQUAL(result.IsTransportError(), false); - } - }; - - IThreadFactory* pool = SystemThreadFactory(); - - TAtomic finished = 0; - TVector<TAutoPtr<IThreadFactory::IThread>> threads; - threads.resize(10); - for (size_t i = 0; i < threads.size(); i++) { - TString namePrefix; - namePrefix.append(5000, 'a' + i); - threads[i] = pool->Run([threadFunc, namePrefix, &finished]() { - threadFunc(namePrefix); - AtomicIncrement(finished); - }); - } - - // Wait for split to happen - while (AtomicGet(finished) < (i64)threads.size()) { - size_t shardsAfter = oldClient.GetTablePartitions("/Root/Foo/TimestampIndex/indexImplTable").size(); - if (shardsAfter > shardsBefore) { - AtomicSet(enough, 1); - break; - } - Sleep(TDuration::Seconds(5)); - } - - for (size_t i = 0; i < threads.size(); i++) { - threads[i]->Join(); - } - - int retries = 5; - size_t shardsAfter = 0; - for (;retries > 0 && shardsAfter <= shardsBefore; --retries, Sleep(TDuration::Seconds(1))) { - shardsAfter = oldClient.GetTablePartitions("/Root/Foo/TimestampIndex/indexImplTable").size(); - } - Cerr << "Index table has " << shardsAfter << " shards" << Endl; - UNIT_ASSERT_C(shardsAfter > shardsBefore, "Index table didn't split!!11 O_O"); - } - Y_UNIT_TEST(AlterIndexImplBySuperUser) { TKikimrWithGrpcAndRootSchema server; |