aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorkungurtsev <kungasc@ydb.tech>2025-04-24 10:15:42 +0200
committerGitHub <noreply@github.com>2025-04-24 10:15:42 +0200
commitf840fd0f0e36ab01c1a44ff60abd59fbec41b3ad (patch)
treeb956a58e18a56f1ba74b333878b463fe6c93c0c4
parentb9cbd36cfb3f65d9ea88104b231f7c699aba501c (diff)
downloadydb-f840fd0f0e36ab01c1a44ff60abd59fbec41b3ad.tar.gz
Lock index impl tables before scanning them (#17229)
-rw-r--r--ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp158
-rw-r--r--ydb/core/protos/flat_scheme_op.proto1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp11
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp10
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp2
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__table_stats.cpp34
-rw-r--r--ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp7
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp223
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp6
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_info_types.h1
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path.cpp4
-rw-r--r--ydb/core/tx/schemeshard/schemeshard_path.h1
13 files changed, 339 insertions, 120 deletions
diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
index 248c9ebb9b4..bcddfee4385 100644
--- a/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
+++ b/ydb/core/kqp/ut/indexes/kqp_indexes_ut.cpp
@@ -119,6 +119,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto& server = kikimr.GetTestServer();
auto gateway = GetIcGateway(server);
@@ -275,6 +277,8 @@ Y_UNIT_TEST_SUITE(KqpIndexMetadata) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto& server = kikimr.GetTestServer();
auto gateway = GetIcGateway(server);
@@ -389,6 +393,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -447,6 +453,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -527,6 +535,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -607,6 +617,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -792,6 +804,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -915,6 +929,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1047,6 +1063,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1282,6 +1300,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({ setting });
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1377,6 +1397,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1514,6 +1536,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1707,6 +1731,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1797,6 +1823,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -1863,6 +1891,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
TScriptingClient client(kikimr.GetDriver());
{
@@ -1904,6 +1934,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -2382,6 +2414,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, false);
{
@@ -2425,6 +2459,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, false);
{
@@ -2468,6 +2504,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, true);
{
@@ -2511,6 +2549,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, true);
{
@@ -2554,6 +2594,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, false);
{
@@ -2597,6 +2639,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, false);
{
@@ -2640,6 +2684,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, true);
{
@@ -2683,6 +2729,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForVectorIndex(db, true);
{
@@ -2716,6 +2764,52 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
DoPositiveQueriesVectorIndexOrderByCosine(session);
}
+ Y_UNIT_TEST(SimpleVectorIndexOrderByCosineDistanceNotNullableLevel3) {
+ NKikimrConfig::TFeatureFlags featureFlags;
+ featureFlags.SetEnableVectorIndex(true);
+ auto setting = NKikimrKqp::TKqpSetting();
+ auto serverSettings = TKikimrSettings()
+ .SetFeatureFlags(featureFlags)
+ .SetKqpSettings({setting});
+
+ TKikimrRunner kikimr(serverSettings);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
+ auto db = kikimr.GetTableClient();
+ auto session = DoCreateTableForVectorIndex(db, false);
+ {
+ const TString createIndex(Q_(R"(
+ ALTER TABLE `/Root/TestTable`
+ ADD INDEX index
+ GLOBAL USING vector_kmeans_tree
+ ON (emb)
+ WITH (distance=cosine, vector_type="uint8", vector_dimension=2, levels=3, clusters=2);
+ )"));
+
+ auto result = session.ExecuteSchemeQuery(createIndex)
+ .ExtractValueSync();
+
+ UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
+ }
+ {
+ auto result = session.DescribeTable("/Root/TestTable").ExtractValueSync();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
+ const auto& indexes = result.GetTableDescription().GetIndexDescriptions();
+ UNIT_ASSERT_EQUAL(indexes.size(), 1);
+ UNIT_ASSERT_EQUAL(indexes[0].GetIndexName(), "index");
+ UNIT_ASSERT_EQUAL(indexes[0].GetIndexColumns(), std::vector<std::string>{"emb"});
+ const auto& settings = std::get<TKMeansTreeSettings>(indexes[0].GetIndexSettings());
+ UNIT_ASSERT_EQUAL(settings.Settings.Metric, NYdb::NTable::TVectorIndexSettings::EMetric::CosineDistance);
+ UNIT_ASSERT_EQUAL(settings.Settings.VectorType, NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8);
+ UNIT_ASSERT_EQUAL(settings.Settings.VectorDimension, 2);
+ UNIT_ASSERT_EQUAL(settings.Levels, 3);
+ UNIT_ASSERT_EQUAL(settings.Clusters, 2);
+ }
+ // TODO: fix somehow?
+ // DoPositiveQueriesVectorIndexOrderByCosine(session);
+ }
+
void DoPositiveQueriesPrefixedVectorIndexOrderBy(
TSession& session,
std::string_view function,
@@ -2888,6 +2982,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, false);
@@ -2933,6 +3029,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, false);
@@ -2978,6 +3076,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, true);
@@ -3023,6 +3123,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, true);
@@ -3068,6 +3170,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, false);
@@ -3205,6 +3309,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, false);
@@ -3250,6 +3356,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, true);
@@ -3295,6 +3403,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = DoCreateTableForPrefixedVectorIndex(db, true);
@@ -3408,6 +3518,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -3485,6 +3597,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -3636,6 +3750,8 @@ Y_UNIT_TEST_SUITE(KqpIndexes) {
.SetAppConfig(app);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -3714,6 +3830,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -3762,6 +3880,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -3826,6 +3946,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -4017,6 +4139,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -4144,6 +4268,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(app);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -4484,6 +4610,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(app);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -4760,6 +4888,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -4998,6 +5128,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5160,6 +5292,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5247,6 +5381,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto scheme = kikimr.GetSchemeClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5297,6 +5433,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5337,6 +5475,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5424,6 +5564,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5622,6 +5764,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
serverSettings.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTablesWithIndex(session);
@@ -5698,6 +5842,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
CreateSampleTablesWithIndex(session);
@@ -5775,6 +5921,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -5965,6 +6113,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
TScriptingClient client(kikimr.GetDriver());
auto scriptResult = client.ExecuteYqlScript(R"(
@@ -6058,6 +6208,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6115,6 +6267,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6166,6 +6320,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetKqpSettings({setting});
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
@@ -6712,6 +6868,8 @@ R"([[#;#;["Primary1"];[41u]];[["Secondary2"];[2u];["Primary2"];[42u]];[["Seconda
.SetAppConfig(appConfig);
TKikimrRunner kikimr(serverSettings);
kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::BUILD_INDEX, NActors::NLog::PRI_TRACE);
+ kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_TRACE);
+
auto db = kikimr.GetTableClient();
auto session = db.CreateSession().GetValueSync().GetSession();
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
index 41176fb96b2..fcd3c0adbf2 100644
--- a/ydb/core/protos/flat_scheme_op.proto
+++ b/ydb/core/protos/flat_scheme_op.proto
@@ -1587,6 +1587,7 @@ message TIndexBuildControl {
message TLockConfig {
optional string Name = 1;
+ optional uint64 LockTxId = 2; // if missing, current tx id is used
}
message TLockGuard {
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp
index 06a5c1fa3e6..ce8f4a7c543 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_apply_build_index.cpp
@@ -30,7 +30,7 @@ ISubOperation::TPtr FinalizeIndexImplTable(TOperationContext& context, const TPa
return CreateFinalizeBuildIndexImplTable(partId, transaction);
}
-ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& nextId, const TOperationId& partId, const TString& name, const TPathId& pathId, bool& rejected) {
+ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& nextId, const TOperationId& partId, const TString& name, const TPathId& pathId, const NKikimrSchemeOp::TLockGuard& lockGuard, bool& rejected) {
TPath implTable = index.Child(name);
Y_ABORT_UNLESS(implTable->PathId == pathId);
Y_ABORT_UNLESS(implTable.LeafName() == name);
@@ -48,6 +48,11 @@ ISubOperation::TPtr DropIndexImplTable(const TPath& index, const TOperationId& n
}
rejected = false;
auto transaction = TransactionTemplate(index.PathString(), NKikimrSchemeOp::EOperationType::ESchemeOpDropTable);
+ if (implTable.IsLocked()) {
+ // because some impl tables may be not locked, do not pass lock guard for them
+ // otherwise `CheckLocks` check would fail
+ *transaction.MutableLockGuard() = lockGuard;
+ }
auto operation = transaction.MutableDrop();
operation->SetName(name);
return CreateDropTable(partId, transaction);
@@ -98,7 +103,7 @@ TVector<ISubOperation::TPtr> ApplyBuildIndex(TOperationId nextId, const TTxTrans
const auto partId = NextPartId(nextId, result);
if (NTableIndex::IsBuildImplTable(indexImplTableName)) {
bool rejected = false;
- auto op = DropIndexImplTable(index, nextId, partId, indexImplTableName, indexChildItems.second, rejected);
+ auto op = DropIndexImplTable(index, nextId, partId, indexImplTableName, indexChildItems.second, tx.GetLockGuard(), rejected);
if (rejected) {
return {std::move(op)};
}
@@ -153,7 +158,7 @@ TVector<ISubOperation::TPtr> CancelBuildIndex(TOperationId nextId, const TTxTran
for (auto& indexChildItems : index.Base()->GetChildren()) {
const auto partId = NextPartId(nextId, result);
bool rejected = false;
- auto op = DropIndexImplTable(index, nextId, partId, indexChildItems.first, indexChildItems.second, rejected);
+ auto op = DropIndexImplTable(index, nextId, partId, indexChildItems.first, indexChildItems.second, tx.GetLockGuard(), rejected);
if (rejected) {
return {std::move(op)};
}
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp
index e8c02b30c81..07efe6838fa 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_create_lock.cpp
@@ -95,6 +95,9 @@ public:
THolder<TProposeResponse> Propose(const TString&, TOperationContext& context) override {
const auto& workingDir = Transaction.GetWorkingDir();
const auto& op = Transaction.GetLockConfig();
+ const TTxId lockTxId = op.HasLockTxId()
+ ? TTxId(op.GetLockTxId())
+ : OperationId.GetTxId();
LOG_N("TCreateLock Propose"
<< ": opId# " << OperationId
@@ -158,13 +161,12 @@ public:
const auto pathId = tablePath.Base()->PathId;
result->SetPathId(pathId.LocalPathId);
- if (tablePath.LockedBy() == OperationId.GetTxId()) {
+ if (tablePath.LockedBy() == lockTxId) {
result->SetError(NKikimrScheme::StatusAlreadyExists, TStringBuilder() << "path checks failed"
<< ", path already locked by this operation"
<< ", path: " << tablePath.PathString());
return result;
}
-
TString errStr;
if (!context.SS->CheckLocks(pathId, Transaction, errStr)) {
result->SetError(NKikimrScheme::StatusMultipleModifications, errStr);
@@ -177,7 +179,7 @@ public:
context.MemChanges.GrabNewTxState(context.SS, OperationId);
context.DbChanges.PersistPath(pathId);
- context.DbChanges.PersistLongLock(pathId, OperationId.GetTxId());
+ context.DbChanges.PersistLongLock(pathId, lockTxId);
context.DbChanges.PersistTxState(OperationId);
Y_ABORT_UNLESS(!context.SS->FindTx(OperationId));
@@ -194,7 +196,7 @@ public:
context.OnComplete.Dependence(splitOpId.GetTxId(), OperationId.GetTxId());
}
- context.SS->LockedPaths[pathId] = OperationId.GetTxId();
+ context.SS->LockedPaths[pathId] = lockTxId;
context.SS->TabletCounters->Simple()[COUNTER_LOCKS_COUNT].Add(1);
context.OnComplete.ActivateTx(OperationId);
diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp
index 8fdfeb0a7d0..4e3cc7c7931 100644
--- a/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__operation_drop_lock.cpp
@@ -164,7 +164,7 @@ public:
const auto pathId = dstPath.Base()->PathId;
result->SetPathId(pathId.LocalPathId);
- if (!dstPath.LockedBy()) {
+ if (!dstPath.IsLocked()) {
result->SetError(TEvSchemeShard::EStatus::StatusAlreadyExists, TStringBuilder() << "path checks failed"
<< ", path already unlocked"
<< ", path: " << dstPath.PathString());
diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
index 9dfaac17560..61f3bb7a082 100644
--- a/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__table_stats.cpp
@@ -484,22 +484,18 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
return true;
}
- {
- auto path = TPath::Init(pathId, Self);
- auto checks = path.Check();
-
- constexpr ui64 deltaShards = 2;
- checks
- .PathShardsLimit(deltaShards)
- .ShardsLimit(deltaShards);
-
- if (!checks) {
- LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
- "Do not request full stats from datashard"
- << ", datashard: " << datashardId
- << ", reason: " << checks.GetError());
- return true;
- }
+ auto path = TPath::Init(pathId, Self);
+ auto checks = path.Check();
+ constexpr ui64 deltaShards = 2;
+ checks
+ .PathShardsLimit(deltaShards)
+ .ShardsLimit(deltaShards);
+ if (!checks) {
+ LOG_NOTICE_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
+ "Do not request full stats from datashard"
+ << ", datashard: " << datashardId
+ << ", reason: " << checks.GetError());
+ return true;
}
if (newStats.HasBorrowedData) {
@@ -509,6 +505,12 @@ bool TTxStoreTableStats::PersistSingleStats(const TPathId& pathId,
return true;
}
+ if (path.IsLocked()) {
+ LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
+ "Postpone split tablet " << datashardId << " because it is locked by " << path.LockedBy());
+ return true;
+ }
+
// Request histograms from the datashard
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
"Requesting full tablet stats " << datashardId << " to split it");
diff --git a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp
index f78ff977165..7ec9bc887f4 100644
--- a/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard__table_stats_histogram.cpp
@@ -341,6 +341,7 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex
}
TTableInfo::TPtr table = Self->Tables[tableId];
+ auto path = TPath::Init(tableId, Self);
if (!Self->TabletIdToShardIdx.contains(datashardId)) {
LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
@@ -355,6 +356,12 @@ bool TTxPartitionHistogram::Execute(TTransactionContext& txc, const TActorContex
return true;
}
+ if (path.IsLocked()) {
+ LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD,
+ "TTxPartitionHistogram Skip locked table tablet " << datashardId << " by " << path.LockedBy());
+ return true;
+ }
+
auto shardIdx = Self->TabletIdToShardIdx[datashardId];
const auto forceShardSplitSettings = Self->SplitSettings.GetForceShardSplitSettings();
diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
index 7e2df90002e..f06974855a0 100644
--- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp
@@ -22,45 +22,6 @@
namespace NKikimr {
namespace NSchemeShard {
-static constexpr const char* Name(TIndexBuildInfo::EState state) noexcept {
- switch (state) {
- case TIndexBuildInfo::EState::Invalid:
- return "Invalid";
- case TIndexBuildInfo::EState::AlterMainTable:
- return "AlterMainTable";
- case TIndexBuildInfo::EState::Locking:
- return "Locking";
- case TIndexBuildInfo::EState::GatheringStatistics:
- return "GatheringStatistics";
- case TIndexBuildInfo::EState::Initiating:
- return "Initiating";
- case TIndexBuildInfo::EState::Filling:
- return "Filling";
- case TIndexBuildInfo::EState::DropBuild:
- return "DropBuild";
- case TIndexBuildInfo::EState::CreateBuild:
- return "CreateBuild";
- case TIndexBuildInfo::EState::Applying:
- return "Applying";
- case TIndexBuildInfo::EState::Unlocking:
- return "Unlocking";
- case TIndexBuildInfo::EState::Done:
- return "Done";
- case TIndexBuildInfo::EState::Cancellation_Applying:
- return "Cancellation_Applying";
- case TIndexBuildInfo::EState::Cancellation_Unlocking:
- return "Cancellation_Unlocking";
- case TIndexBuildInfo::EState::Cancelled:
- return "Cancelled";
- case TIndexBuildInfo::EState::Rejection_Applying:
- return "Rejection_Applying";
- case TIndexBuildInfo::EState::Rejection_Unlocking:
- return "Rejection_Unlocking";
- case TIndexBuildInfo::EState::Rejected:
- return "Rejected";
- }
-}
-
// return count, parts, step
static std::tuple<NTableIndex::TClusterId, NTableIndex::TClusterId, NTableIndex::TClusterId> ComputeKMeansBoundaries(const NSchemeShard::TTableInfo& tableInfo, const TIndexBuildInfo& buildInfo) {
const auto& kmeans = buildInfo.KMeans;
@@ -244,24 +205,32 @@ private:
}
};
+TPath GetBuildPath(TSchemeShard* ss, const TIndexBuildInfo& buildInfo, const TString& tableName) {
+ return TPath::Init(buildInfo.TablePathId, ss)
+ .Dive(buildInfo.IndexName)
+ .Dive(tableName);
+}
+
THolder<TEvSchemeShard::TEvModifySchemeTransaction> LockPropose(
- TSchemeShard* ss, const TIndexBuildInfo& buildInfo)
+ TSchemeShard* ss, const TIndexBuildInfo& buildInfo, TTxId txId, const TPath& path)
{
- auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(buildInfo.LockTxId), ss->TabletID());
+ auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(txId), ss->TabletID());
propose->Record.SetFailOnExist(false);
NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction();
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateLock);
modifyScheme.SetInternal(true);
-
- TPath path = TPath::Init(buildInfo.TablePathId, ss);
modifyScheme.SetWorkingDir(path.Parent().PathString());
modifyScheme.MutableLockConfig()->SetName(path.LeafName());
+ modifyScheme.MutableLockConfig()->SetLockTxId(ui64(buildInfo.LockTxId));
+
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "LockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
return propose;
}
-THolder<TEvSchemeShard::TEvModifySchemeTransaction> InitiatePropose(
+THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateIndexPropose(
TSchemeShard* ss, const TIndexBuildInfo& buildInfo)
{
auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(buildInfo.InitiateTxId), ss->TabletID());
@@ -279,9 +248,12 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> InitiatePropose(
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateColumnBuild);
buildInfo.SerializeToProto(ss, modifyScheme.MutableInitiateColumnBuild());
} else {
- Y_ABORT("Unknown operation kind while building InitiatePropose");
+ Y_ABORT("Unknown operation kind while building CreateIndexPropose");
}
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "CreateIndexPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
@@ -293,14 +265,22 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> DropBuildPropose(
auto propose = MakeHolder<TEvSchemeShard::TEvModifySchemeTransaction>(ui64(buildInfo.ApplyTxId), ss->TabletID());
propose->Record.SetFailOnExist(true);
- auto path = TPath::Init(buildInfo.TablePathId, ss).Dive(buildInfo.IndexName);
+ auto path = GetBuildPath(ss, buildInfo, buildInfo.KMeans.WriteTo(true));
NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction();
modifyScheme.SetInternal(true);
- modifyScheme.SetWorkingDir(path.PathString());
+ modifyScheme.SetWorkingDir(path.Parent().PathString());
+ if (path.IsLocked()) {
+ // because some impl tables may be not locked, do not pass lock guard for them
+ // otherwise `CheckLocks` check would fail
+ modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId));
+ }
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropTable);
- modifyScheme.MutableDrop()->SetName(buildInfo.KMeans.WriteTo(true));
+ modifyScheme.MutableDrop()->SetName(path->Name);
+
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "DropBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
return propose;
}
@@ -333,10 +313,6 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateBuildPropose(
modifyScheme.ClearInitiateIndexBuild();
}
- // TODO(mbkkt) for levels greater than zero we need to disable split/merge completely
- // For now it's not guranteed, but very likely
- // But lock is really unconvinient approach (needs to store TxId/etc)
- // So maybe best way to do this is specify something in defintion, that will prevent these operations like IsBackup
using namespace NTableIndex::NTableVectorKmeansTreeIndex;
modifyScheme.SetWorkingDir(path.Dive(buildInfo.IndexName).PathString());
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpInitiateBuildIndexImplTable);
@@ -361,6 +337,10 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateBuildPropose(
const auto shards = tableInfo->GetShard2PartitionIdx().size();
policy.SetMinPartitionsCount(shards);
policy.SetMaxPartitionsCount(shards);
+
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
op = CalcVectorKmeansTreePostingImplTableDesc({}, tableInfo, tableInfo->PartitionConfig(), implTableColumns, {}, suffix);
@@ -381,6 +361,10 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CreateBuildPropose(
}
policy.SetMinPartitionsCount(op.SplitBoundarySize() + 1);
policy.SetMaxPartitionsCount(op.SplitBoundarySize() + 1);
+
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "CreateBuildPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
@@ -426,6 +410,9 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> AlterMainTablePropose(
}
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "AlterMainTablePropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
@@ -438,9 +425,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> ApplyPropose(
NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction();
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpApplyIndexBuild);
modifyScheme.SetInternal(true);
-
modifyScheme.SetWorkingDir(TPath::Init(buildInfo.DomainPathId, ss).PathString());
-
modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId));
auto& indexBuild = *modifyScheme.MutableApplyIndexBuild();
@@ -453,6 +438,9 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> ApplyPropose(
indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId));
indexBuild.SetBuildIndexId(ui64(buildInfo.Id));
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "ApplyPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
@@ -465,7 +453,6 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> UnlockPropose(
NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction();
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpDropLock);
modifyScheme.SetInternal(true);
-
modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId));
TPath path = TPath::Init(buildInfo.TablePathId, ss);
@@ -474,6 +461,9 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> UnlockPropose(
auto& lockConfig = *modifyScheme.MutableLockConfig();
lockConfig.SetName(path.LeafName());
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "UnlockPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
@@ -486,9 +476,7 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CancelPropose(
NKikimrSchemeOp::TModifyScheme& modifyScheme = *propose->Record.AddTransaction();
modifyScheme.SetOperationType(NKikimrSchemeOp::ESchemeOpCancelIndexBuild);
modifyScheme.SetInternal(true);
-
modifyScheme.SetWorkingDir(TPath::Init(buildInfo.DomainPathId, ss).PathString());
-
modifyScheme.MutableLockGuard()->SetOwnerTxId(ui64(buildInfo.LockTxId));
auto& indexBuild = *modifyScheme.MutableCancelIndexBuild();
@@ -497,6 +485,9 @@ THolder<TEvSchemeShard::TEvModifySchemeTransaction> CancelPropose(
indexBuild.SetSnapshotTxId(ui64(buildInfo.InitiateTxId));
indexBuild.SetBuildIndexId(ui64(buildInfo.Id));
+ LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::BUILD_INDEX,
+ "CancelPropose " << buildInfo.Id << " " << buildInfo.State << " " << propose->Record.ShortDebugString());
+
return propose;
}
@@ -543,8 +534,8 @@ private:
if (buildInfo.KMeans.Level == 1) {
buildInfo.TablePathId.ToProto(ev->Record.MutablePathId());
} else {
- auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName);
- path.Dive(buildInfo.KMeans.ReadFrom())->PathId.ToProto(ev->Record.MutablePathId());
+ auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom());
+ path->PathId.ToProto(ev->Record.MutablePathId());
}
ev->Record.SetK(buildInfo.KMeans.K);
@@ -666,8 +657,8 @@ private:
auto ev = MakeHolder<TEvDataShard::TEvPrefixKMeansRequest>();
ev->Record.SetId(ui64(BuildId));
- auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName);
- path.Dive(buildInfo.KMeans.ReadFrom())->PathId.ToProto(ev->Record.MutablePathId());
+ auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom());
+ path->PathId.ToProto(ev->Record.MutablePathId());
path.Rise();
*ev->Record.MutableSettings() = std::get<NKikimrSchemeOp::TVectorIndexKmeansTreeDescription>(
buildInfo.SpecializedIndexDescription).GetSettings().settings();
@@ -710,7 +701,7 @@ private:
buildInfo.SerializeToProto(Self, ev->Record.MutableColumnBuildSettings());
} else {
if (buildInfo.TargetName.empty()) {
- TPath implTable = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName).Dive(
+ TPath implTable = GetBuildPath(Self, buildInfo,
buildInfo.IsBuildPrefixedVectorIndex() ? buildInfo.KMeans.WriteTo() : NTableIndex::ImplTable);
buildInfo.TargetName = implTable.PathString();
@@ -752,9 +743,7 @@ private:
void SendUploadSampleKRequest(TIndexBuildInfo& buildInfo) {
buildInfo.Sample.MakeStrictTop(buildInfo.KMeans.K);
- auto path = TPath::Init(buildInfo.TablePathId, Self)
- .Dive(buildInfo.IndexName)
- .Dive(NTableIndex::NTableVectorKmeansTreeIndex::LevelTable);
+ auto path = GetBuildPath(Self, buildInfo, NTableIndex::NTableVectorKmeansTreeIndex::LevelTable);
Y_ASSERT(buildInfo.Sample.Rows.size() <= buildInfo.KMeans.K);
auto actor = new TUploadSampleK(path.PathString(),
buildInfo.ScanSettings, Self->SelfId(), ui64(BuildId),
@@ -1084,7 +1073,9 @@ private:
}
if (buildInfo.Shards.empty()) {
NIceDb::TNiceDb db(txc.DB);
- InitiateShards(db, buildInfo);
+ if (!InitiateShards(db, buildInfo)) {
+ return false;
+ }
}
switch (buildInfo.BuildKind) {
case TIndexBuildInfo::EBuildKind::BuildSecondaryIndex:
@@ -1111,8 +1102,7 @@ public:
Y_ABORT_UNLESS(buildInfoPtr);
auto& buildInfo = *buildInfoPtr->Get();
- LOG_I("TTxBuildProgress: Resume: id# " << BuildId);
- LOG_D("TTxBuildProgress: Resume: " << buildInfo);
+ LOG_I("TTxBuildProgress: Execute: " << BuildId << " " << buildInfo.State << " " << buildInfo);
switch (buildInfo.State) {
case TIndexBuildInfo::EState::Invalid:
@@ -1120,7 +1110,7 @@ public:
case TIndexBuildInfo::EState::AlterMainTable:
if (buildInfo.AlterMainTableTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.AlterMainTableTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), AlterMainTablePropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.AlterMainTableTxDone) {
@@ -1132,9 +1122,9 @@ public:
break;
case TIndexBuildInfo::EState::Locking:
if (buildInfo.LockTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.LockTxStatus == NKikimrScheme::StatusSuccess) {
- Send(Self->SelfId(), LockPropose(Self, buildInfo), 0, ui64(BuildId));
+ Send(Self->SelfId(), LockPropose(Self, buildInfo, buildInfo.LockTxId, TPath::Init(buildInfo.TablePathId, Self)), 0, ui64(BuildId));
} else if (!buildInfo.LockTxDone) {
Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(buildInfo.LockTxId)));
} else {
@@ -1149,9 +1139,9 @@ public:
break;
case TIndexBuildInfo::EState::Initiating:
if (buildInfo.InitiateTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.InitiateTxStatus == NKikimrScheme::StatusSuccess) {
- Send(Self->SelfId(), InitiatePropose(Self, buildInfo), 0, ui64(BuildId));
+ Send(Self->SelfId(), CreateIndexPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.InitiateTxDone) {
Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(buildInfo.InitiateTxId)));
} else {
@@ -1184,7 +1174,7 @@ public:
Y_ASSERT(buildInfo.IsBuildVectorIndex());
Y_ASSERT(buildInfo.KMeans.Level > 2);
if (buildInfo.ApplyTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), DropBuildPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.ApplyTxDone) {
@@ -1206,7 +1196,7 @@ public:
case TIndexBuildInfo::EState::CreateBuild:
Y_ASSERT(buildInfo.IsBuildVectorIndex());
if (buildInfo.ApplyTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), CreateBuildPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.ApplyTxDone) {
@@ -1228,9 +1218,31 @@ public:
Progress(BuildId);
}
break;
+ case TIndexBuildInfo::EState::LockBuild:
+ Y_ASSERT(buildInfo.IsBuildVectorIndex());
+ if (buildInfo.ApplyTxId == InvalidTxId) {
+ AllocateTxId(BuildId);
+ } else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) {
+ Send(Self->SelfId(), LockPropose(Self, buildInfo, buildInfo.ApplyTxId, GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom())), 0, ui64(BuildId));
+ } else if (!buildInfo.ApplyTxDone) {
+ Send(Self->SelfId(), MakeHolder<TEvSchemeShard::TEvNotifyTxCompletion>(ui64(buildInfo.ApplyTxId)));
+ } else {
+ buildInfo.ApplyTxId = {};
+ buildInfo.ApplyTxStatus = NKikimrScheme::StatusSuccess;
+ buildInfo.ApplyTxDone = false;
+
+ NIceDb::TNiceDb db(txc.DB);
+ Self->PersistBuildIndexApplyTxId(db, buildInfo);
+ Self->PersistBuildIndexApplyTxStatus(db, buildInfo);
+ Self->PersistBuildIndexApplyTxDone(db, buildInfo);
+
+ ChangeState(BuildId, TIndexBuildInfo::EState::Filling);
+ Progress(BuildId);
+ }
+ break;
case TIndexBuildInfo::EState::Applying:
if (buildInfo.ApplyTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), ApplyPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.ApplyTxDone) {
@@ -1242,7 +1254,7 @@ public:
break;
case TIndexBuildInfo::EState::Unlocking:
if (buildInfo.UnlockTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.UnlockTxDone) {
@@ -1258,7 +1270,7 @@ public:
break;
case TIndexBuildInfo::EState::Cancellation_Applying:
if (buildInfo.ApplyTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.ApplyTxDone) {
@@ -1270,7 +1282,7 @@ public:
break;
case TIndexBuildInfo::EState::Cancellation_Unlocking:
if (buildInfo.UnlockTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.UnlockTxDone) {
@@ -1286,7 +1298,7 @@ public:
break;
case TIndexBuildInfo::EState::Rejection_Applying:
if (buildInfo.ApplyTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.ApplyTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), CancelPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.ApplyTxDone) {
@@ -1298,7 +1310,7 @@ public:
break;
case TIndexBuildInfo::EState::Rejection_Unlocking:
if (buildInfo.UnlockTxId == InvalidTxId) {
- Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(BuildId));
+ AllocateTxId(BuildId);
} else if (buildInfo.UnlockTxStatus == NKikimrScheme::StatusSuccess) {
Send(Self->SelfId(), UnlockPropose(Self, buildInfo), 0, ui64(BuildId));
} else if (!buildInfo.UnlockTxDone) {
@@ -1332,7 +1344,7 @@ public:
return TSerializedTableRange{{&from, 1}, false, {&to, 1}, true};
}
- void InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo& buildInfo) {
+ bool InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo& buildInfo) {
LOG_D("InitiateShards " << buildInfo.DebugString());
Y_ASSERT(buildInfo.Shards.empty());
@@ -1344,8 +1356,15 @@ public:
if (buildInfo.KMeans.Level == 1) {
table = Self->Tables.at(buildInfo.TablePathId);
} else {
- auto path = TPath::Init(buildInfo.TablePathId, Self).Dive(buildInfo.IndexName);
- table = Self->Tables.at(path.Dive(buildInfo.KMeans.ReadFrom())->PathId);
+ auto path = GetBuildPath(Self, buildInfo, buildInfo.KMeans.ReadFrom());
+ table = Self->Tables.at(path->PathId);
+
+ if (!path.IsLocked()) { // lock is needed to prevent table shards from being split
+ ChangeState(buildInfo.Id, TIndexBuildInfo::EState::LockBuild);
+ Progress(buildInfo.Id);
+ return false;
+ }
+ Y_ASSERT(path.LockedBy() == buildInfo.LockTxId);
}
auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns
TSerializedTableRange shardRange = InfiniteRange(tableColumns.Keys.size());
@@ -1366,6 +1385,8 @@ public:
Self->PersistBuildIndexUploadInitiate(db, BuildId, x.ShardIdx, it->second);
}
+
+ return true;
}
void DoComplete(const TActorContext& ctx) override {
@@ -1483,10 +1504,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -1637,10 +1659,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -1762,10 +1785,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -1886,10 +1910,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -2011,10 +2036,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -2093,10 +2119,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -2246,10 +2273,11 @@ public:
case TIndexBuildInfo::EState::Initiating:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Unlocking:
case TIndexBuildInfo::EState::Done:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Cancellation_Unlocking:
case TIndexBuildInfo::EState::Cancelled:
@@ -2325,6 +2353,7 @@ public:
}
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Rejection_Applying:
@@ -2351,7 +2380,7 @@ public:
case TIndexBuildInfo::EState::Done:
case TIndexBuildInfo::EState::Cancelled:
case TIndexBuildInfo::EState::Rejected:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
}
Progress(buildId);
@@ -2432,7 +2461,7 @@ public:
if (statusCode != Ydb::StatusIds::SUCCESS) {
buildInfo.Issue += TStringBuilder()
- << "At " << Name(state) << " state got unsuccess propose result"
+ << "At " << state << " state got unsuccess propose result"
<< ", status: " << NKikimrScheme::EStatus_Name(record.GetStatus())
<< ", reason: " << record.GetReason();
Self->PersistBuildIndexIssue(db, buildInfo);
@@ -2451,7 +2480,7 @@ public:
// no op
} else {
buildInfo.Issue += TStringBuilder()
- << "At " << Name(state) << " state got unsuccess propose result"
+ << "At " << state << " state got unsuccess propose result"
<< ", status: " << NKikimrScheme::EStatus_Name(record.GetStatus())
<< ", reason: " << record.GetReason();
Self->PersistBuildIndexIssue(db, buildInfo);
@@ -2492,6 +2521,7 @@ public:
}
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Rejection_Applying:
{
@@ -2549,7 +2579,7 @@ public:
case TIndexBuildInfo::EState::Done:
case TIndexBuildInfo::EState::Cancelled:
case TIndexBuildInfo::EState::Rejected:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
}
Progress(buildId);
@@ -2606,6 +2636,7 @@ public:
break;
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
case TIndexBuildInfo::EState::Applying:
case TIndexBuildInfo::EState::Cancellation_Applying:
case TIndexBuildInfo::EState::Rejection_Applying:
@@ -2628,7 +2659,7 @@ public:
case TIndexBuildInfo::EState::Done:
case TIndexBuildInfo::EState::Cancelled:
case TIndexBuildInfo::EState::Rejected:
- Y_FAIL_S("Unreachable " << Name(state));
+ Y_FAIL_S("Unreachable " << state);
}
Progress(buildId);
diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp
index e07d231c38e..9a3eecc5076 100644
--- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp
@@ -177,6 +177,11 @@ void TSchemeShard::TIndexBuilder::TTxBase::Send(TActorId dst, THolder<IEventBase
SideEffects.Send(dst, message.Release(), cookie, flags);
}
+void TSchemeShard::TIndexBuilder::TTxBase::AllocateTxId(TIndexBuildId buildId) {
+ LOG_D("AllocateTxId " << buildId);
+ Send(Self->TxAllocatorClient, MakeHolder<TEvTxAllocatorClient::TEvAllocate>(), 0, ui64(buildId));
+}
+
void TSchemeShard::TIndexBuilder::TTxBase::ChangeState(TIndexBuildId id, TIndexBuildInfo::EState state) {
StateChanges.push_back(TChangeStateRec(id, state));
}
@@ -215,6 +220,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild
case TIndexBuildInfo::EState::Filling:
case TIndexBuildInfo::EState::DropBuild:
case TIndexBuildInfo::EState::CreateBuild:
+ case TIndexBuildInfo::EState::LockBuild:
index.SetState(Ydb::Table::IndexBuildState::STATE_TRANSFERING_DATA);
index.SetProgress(indexInfo.CalcProgressPercent());
break;
diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h
index 1a70dd31ec5..02f96b34aef 100644
--- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h
+++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.h
@@ -32,6 +32,7 @@ private:
protected:
void Send(TActorId dst, THolder<IEventBase> message, ui32 flags = 0, ui64 cookie = 0);
+ void AllocateTxId(TIndexBuildId buildId);
void ChangeState(TIndexBuildId id, TIndexBuildInfo::EState state);
void Progress(TIndexBuildId id);
void Fill(NKikimrIndexBuilder::TIndexBuild& index, const TIndexBuildInfo& indexInfo);
diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h
index 50f489a1e90..d6b90d7e3bc 100644
--- a/ydb/core/tx/schemeshard/schemeshard_info_types.h
+++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h
@@ -3040,6 +3040,7 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> {
Filling = 40,
DropBuild = 45,
CreateBuild = 46,
+ LockBuild = 47,
Applying = 50,
Unlocking = 60,
Done = 200,
diff --git a/ydb/core/tx/schemeshard/schemeshard_path.cpp b/ydb/core/tx/schemeshard/schemeshard_path.cpp
index 01bae14754e..c62f7be2d41 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path.cpp
+++ b/ydb/core/tx/schemeshard/schemeshard_path.cpp
@@ -1875,6 +1875,10 @@ ui64 TPath::GetEffectiveACLVersion() const {
return version;
}
+bool TPath::IsLocked() const {
+ return SS->LockedPaths.contains(Base()->PathId);
+}
+
TTxId TPath::LockedBy() const {
auto it = SS->LockedPaths.find(Base()->PathId);
if (it != SS->LockedPaths.end()) {
diff --git a/ydb/core/tx/schemeshard/schemeshard_path.h b/ydb/core/tx/schemeshard/schemeshard_path.h
index 4e167ac23dc..071e488b7b8 100644
--- a/ydb/core/tx/schemeshard/schemeshard_path.h
+++ b/ydb/core/tx/schemeshard/schemeshard_path.h
@@ -183,6 +183,7 @@ public:
bool IsValidLeafName(TString& explain) const;
TString GetEffectiveACL() const;
ui64 GetEffectiveACLVersion() const;
+ bool IsLocked() const;
TTxId LockedBy() const;
bool IsActive() const;