diff options
author | svc <svc@yandex-team.ru> | 2022-06-06 14:52:19 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-06-06 14:52:19 +0300 |
commit | d664fb5c2f4a6a578c314d2ecee39c904d2b4418 (patch) | |
tree | 66fdf279893409594c4b9aec97baa1ad523055c8 | |
parent | 1da68cb4313e3258a9331a48e67b71a6b9bb5706 (diff) | |
download | ydb-d664fb5c2f4a6a578c314d2ecee39c904d2b4418.tar.gz |
KIKIMR-14517 online index miss lead null records
REVIEW: 2510843
REVIEW: 2512400
x-ydb-stable-ref: 96e93188329f9796d582b17ca6f0901fa419eff3
-rw-r--r-- | ydb/core/kqp/ut/kqp_indexes_ut.cpp | 234 | ||||
-rw-r--r-- | ydb/core/tx/datashard/datashard_ut_range_ops.cpp | 399 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard__init.cpp | 9 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp | 14 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard_info_types.cpp | 12 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/schemeshard_info_types.h | 2 | ||||
-rw-r--r-- | ydb/core/tx/schemeshard/ut_index_build_reboots.cpp | 31 |
7 files changed, 521 insertions, 180 deletions
diff --git a/ydb/core/kqp/ut/kqp_indexes_ut.cpp b/ydb/core/kqp/ut/kqp_indexes_ut.cpp index f928dc73d4..39be9e8e0d 100644 --- a/ydb/core/kqp/ut/kqp_indexes_ut.cpp +++ b/ydb/core/kqp/ut/kqp_indexes_ut.cpp @@ -1463,6 +1463,240 @@ Y_UNIT_TEST_SUITE(KqpIndexes) { } + Y_UNIT_TEST_QUAD(UpsertWithNullKeysSimple, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) + .SetKqpSettings({ setting }); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("IndexColumn", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + { + const TString query1(Q_(R"( + UPSERT INTO `/Root/TestTable` (Key, IndexColumn, Value) VALUES + ("Primary 1", "Secondary 1", "Value 1"), + (Null, "Secondary 2", "Value 2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + TAlterTableSettings alterSettings; + alterSettings.AppendAddIndexes({ TIndexDescription("IndexName", {"IndexColumn"}) }); + auto result = session.AlterTable("/Root/TestTable", alterSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + { + const TString query(Q1_(R"( + SELECT Value FROM `/Root/TestTable` VIEW IndexName WHERE IndexColumn = 'Secondary 2'; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value 2\"]]]"); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/IndexName/indexImplTable"); + const TString expected = + R"([[["Secondary 1"];["Primary 1"]];)" + R"([["Secondary 2"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + const TString query1(Q_(R"( + UPSERT INTO `/Root/TestTable` (Key, IndexColumn, Value) VALUES + ("Primary 3", "Secondary 3", "Value 3"), + (Null, "Secondary 4", "Value 4"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + const TString query(Q1_(R"( + SELECT Value FROM `/Root/TestTable` VIEW IndexName WHERE IndexColumn = 'Secondary 4'; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value 4\"]]]"); + } + { + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/IndexName/indexImplTable"); + const TString expected = + R"([[["Secondary 1"];["Primary 1"]];)" + R"([["Secondary 3"];["Primary 3"]];)" + R"([["Secondary 4"];#]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + } + + Y_UNIT_TEST_QUAD(UpsertWithNullKeysComplex, WithMvcc, UseNewEngine) { + auto setting = NKikimrKqp::TKqpSetting(); + auto serverSettings = TKikimrSettings() + .SetEnableMvcc(WithMvcc) + .SetEnableMvccSnapshotReads(WithMvcc) + .SetKqpSettings({setting}); + TKikimrRunner kikimr(serverSettings); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + // Create table with 1 index + auto tableBuilder = db.GetTableBuilder(); + tableBuilder + .AddNullableColumn("Key", EPrimitiveType::String) + .AddNullableColumn("IndexColumn1", EPrimitiveType::String) + .AddNullableColumn("IndexColumn2", EPrimitiveType::String) + .AddNullableColumn("Value", EPrimitiveType::String); + tableBuilder.SetPrimaryKeyColumns(TVector<TString>{"Key", "IndexColumn1", "IndexColumn2"}); + tableBuilder.AddSecondaryIndex("IndexName1", TVector<TString>{"IndexColumn1"}); + auto result = session.CreateTable("/Root/TestTable", tableBuilder.Build()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + { + // Upsert rows including one with PK starting with Null + const TString query1(Q_(R"( + UPSERT INTO `/Root/TestTable` (Key, IndexColumn1, IndexColumn2, Value) VALUES + ("Primary 1", "Secondary1 1", "Secondary2 1", "Value 1"), + (Null, "Secondary1 2", "Secondary2 2", "Value 2"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + // Read row with PK starting with Null by index1 + const TString query(Q1_(R"( + SELECT Value FROM `/Root/TestTable` VIEW IndexName1 WHERE IndexColumn1 = 'Secondary1 2'; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value 2\"]]]"); + } + { + // Both rows should be in index1 table + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/IndexName1/indexImplTable"); + const TString expected = + R"([[["Secondary1 1"];["Primary 1"];["Secondary2 1"]];)" + R"([["Secondary1 2"];#;["Secondary2 2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + // Add second index with alter + TAlterTableSettings alterSettings; + alterSettings.AppendAddIndexes({ TIndexDescription("IndexName2", {"IndexColumn2"}) }); + auto result = session.AlterTable("/Root/TestTable", alterSettings).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + { + // Read row with PK starting with Null by index2 + const TString query(Q1_(R"( + SELECT Value FROM `/Root/TestTable` VIEW IndexName2 WHERE IndexColumn2 = 'Secondary2 2'; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value 2\"]]]"); + } + { + // Both rows should also be in index2 table + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/IndexName2/indexImplTable"); + const TString expected = + R"([[["Secondary2 1"];["Primary 1"];["Secondary1 1"]];)" + R"([["Secondary2 2"];#;["Secondary1 2"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + // Upsert more rows including one with PK starting with Null + const TString query1(Q_(R"( + UPSERT INTO `/Root/TestTable` (Key, IndexColumn1, IndexColumn2, Value) VALUES + ("Primary 3", "Secondary1 3", "Secondary2 3", "Value 3"), + (Null, "Secondary1 4", "Secondary2 4", "Value 4"); + )")); + + auto result = session.ExecuteDataQuery( + query1, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + } + { + // Read recently added row with PK starting with Null by index2 + const TString query(Q1_(R"( + SELECT Value FROM `/Root/TestTable` VIEW IndexName2 WHERE IndexColumn2 = 'Secondary2 4'; + )")); + + auto result = session.ExecuteDataQuery( + query, + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()) + .ExtractValueSync(); + UNIT_ASSERT(result.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(NYdb::FormatResultSetYson(result.GetResultSet(0)), "[[[\"Value 4\"]]]"); + } + { + // All 4 rows should be in index1 table + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/IndexName1/indexImplTable"); + const TString expected = + R"([[["Secondary1 1"];["Primary 1"];["Secondary2 1"]];)" + R"([["Secondary1 2"];#;["Secondary2 2"]];)" + R"([["Secondary1 3"];["Primary 3"];["Secondary2 3"]];)" + R"([["Secondary1 4"];#;["Secondary2 4"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + { + // All 4 rows should also be in index2 table + const auto& yson = ReadTablePartToYson(session, "/Root/TestTable/IndexName2/indexImplTable"); + const TString expected = + R"([[["Secondary2 1"];["Primary 1"];["Secondary1 1"]];)" + R"([["Secondary2 2"];#;["Secondary1 2"]];)" + R"([["Secondary2 3"];["Primary 3"];["Secondary1 3"]];)" + R"([["Secondary2 4"];#;["Secondary1 4"]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + } + Y_UNIT_TEST_TWIN(KeyIndex, WithMvcc) { auto setting = NKikimrKqp::TKqpSetting(); auto serverSettings = TKikimrSettings() diff --git a/ydb/core/tx/datashard/datashard_ut_range_ops.cpp b/ydb/core/tx/datashard/datashard_ut_range_ops.cpp index 61f0bc9406..681749709b 100644 --- a/ydb/core/tx/datashard/datashard_ut_range_ops.cpp +++ b/ydb/core/tx/datashard/datashard_ut_range_ops.cpp @@ -5,16 +5,18 @@ namespace NKikimr { Y_UNIT_TEST_SUITE(RangeOps) { - TString MakeValue(ui64 value) { - TRawTypeValue raw(&value, sizeof(value), NScheme::NTypeIds::Uint64); - TCell cell(&raw); - TVector<TCell> vec = {cell}; + TString MakePoint(TVector<std::optional<ui64>> values) { + TVector<TCell> vec; + for (const auto& val: values) { + TCell cell = val ? TCell::Make<ui64>(val.value()) : TCell(); + vec.push_back(cell); + } TArrayRef<TCell> cells(vec); return TSerializedCellVec::Serialize(cells); } - TString MakeLeftInf() { - TCell cell; - TVector<TCell> vec = {cell}; + + TString MakeLeftInf(ui32 cellsCount = 1) { + TVector<TCell> vec = TVector(cellsCount, TCell()); TArrayRef<TCell> cells(vec); return TSerializedCellVec::Serialize(cells); } @@ -55,25 +57,100 @@ Y_UNIT_TEST_SUITE(RangeOps) { typeRegistry->CalculateMetadataEtag(); const TVector<NKikimr::NScheme::TTypeId> valueType = {NScheme::NTypeIds::Uint64}; + const TVector<NKikimr::NScheme::TTypeId> pairType = {NScheme::NTypeIds::Uint64, NScheme::NTypeIds::Uint64}; const auto emptyRange = MakeRange( - MakeValue(20), true, - MakeValue(10), true + MakePoint({20}), true, + MakePoint({10}), true ); { //================= + //(-inf; +inf) + //----[bbbbbbbb]------- // range with lead null + + auto first = MakeRange( + MakeLeftInf(pairType.size()), true, + MakeRightInf(), false + ); + + Cerr << "first " << DebugPrintRange(pairType, first.ToTableRange(), *typeRegistry); + Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); + + auto second = MakeRange( + MakePoint({std::nullopt, 1}), true, + MakePoint({20, 20}), true + ); + + Cerr << " second " << DebugPrintRange(pairType, second.ToTableRange(), *typeRegistry); + + auto range = Intersect(pairType, first.ToTableRange(), second.ToTableRange()); + Cerr << " result " << DebugPrintRange(pairType, range, *typeRegistry); + + auto correct = MakeRange( + MakePoint({std::nullopt, 1}), true, + MakePoint({20, 20}), true + ); + + Cerr << " correct " << DebugPrintRange(pairType, correct.ToTableRange(), *typeRegistry); + + CheckRange(pairType, correct.ToTableRange(), range); + Cerr << Endl; + } + + { + // test for KIKIMR-14517 + //================= + //------(-inf; +inf) // -inf is wrongly set up as {null} instead of {null, null} + //----[bbbbbbbb]------- // range with lead null + + // here is a mistake: MakeLeftInf(valueType.size()) is set up as {null} + // but -inf shoud be set up as {null, null} + // however MakePoint({std::nullopt, 1}) is {null, 1} + // as a resul {null} = {null, +inf} > {null, 1} > {null, null} + + auto first = MakeRange( + MakeLeftInf(valueType.size()), true, + MakeRightInf(), false + ); + + Cerr << "first " << DebugPrintRange(pairType, first.ToTableRange(), *typeRegistry); + Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); + + auto second = MakeRange( + MakePoint({std::nullopt, 1}), true, + MakePoint({20, 20}), true + ); + + Cerr << " second " << DebugPrintRange(pairType, second.ToTableRange(), *typeRegistry); + + auto range = Intersect(pairType, first.ToTableRange(), second.ToTableRange()); + Cerr << " result " << DebugPrintRange(pairType, range, *typeRegistry); + + auto correct = MakeRange( + MakePoint({std::nullopt}), true, // {null} = {null, +inf} > {null, 1} > {null, null}, so there is {null} instead {null, 1} + MakePoint({20, 20}), true + ); + + Cerr << " correct " << DebugPrintRange(pairType, correct.ToTableRange(), *typeRegistry); + + CheckRange(pairType, correct.ToTableRange(), range); + Cerr << Endl; + } + + { + //================= //-----------[aaa]--- //----[bbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(5), true + MakePoint({1}), true, + MakePoint({5}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -89,14 +166,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----------[aaa]--- //----[bbbbbb]------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(10), true + MakePoint({1}), true, + MakePoint({10}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -104,8 +181,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(10), true + MakePoint({10}), true, + MakePoint({10}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -120,14 +197,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----------[aaa]--- //----[bbbbbbbb]----- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(15), true + MakePoint({1}), true, + MakePoint({15}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -135,8 +212,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(15), true + MakePoint({10}), true, + MakePoint({15}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -150,14 +227,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----------[aaa]--- //----[bbbbbbbbbb]--- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), true + MakePoint({1}), true, + MakePoint({20}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -165,8 +242,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -180,14 +257,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //---------[aaa]---- //--[bbbbbbbbbbbbb]- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(30), true + MakePoint({1}), true, + MakePoint({30}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -195,8 +272,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -211,14 +288,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //----[aaaaaa]---------- //----[]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(10), true, - MakeValue(10), true + MakePoint({10}), true, + MakePoint({10}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -226,8 +303,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(10), true + MakePoint({10}), true, + MakePoint({10}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -241,14 +318,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //----[aaaaaa]---------- //----[bbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(10), true, - MakeValue(15), true + MakePoint({10}), true, + MakePoint({15}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -256,8 +333,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(15), true + MakePoint({10}), true, + MakePoint({15}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -272,14 +349,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //----[aaaaaa]---------- //----[bbbbbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -287,8 +364,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -303,14 +380,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //----[aaaaaa]---------- //----[bbbbbbbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(10), true, - MakeValue(30), true + MakePoint({10}), true, + MakePoint({30}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -318,8 +395,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -333,14 +410,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----[aaaaaaa]---------- //-------[bbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(15), true, - MakeValue(17), true + MakePoint({15}), true, + MakePoint({17}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -348,8 +425,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(15), true, - MakeValue(17), true + MakePoint({15}), true, + MakePoint({17}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -363,14 +440,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----[aaaaaaa]---------- //-------[bbbbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(15), true, - MakeValue(20), true + MakePoint({15}), true, + MakePoint({20}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -378,8 +455,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(15), true, - MakeValue(20), true + MakePoint({15}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -393,14 +470,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----[aaaaaaa]---------- //-------[bbbbbbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(15), true, - MakeValue(30), true + MakePoint({15}), true, + MakePoint({30}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -408,8 +485,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(15), true, - MakeValue(20), true + MakePoint({15}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -423,14 +500,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----[aaa]---------- //---------[]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(20), true, - MakeValue(20), true + MakePoint({20}), true, + MakePoint({20}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -438,8 +515,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(20), true, - MakeValue(20), true + MakePoint({20}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -453,14 +530,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----[aaa]---------- //---------[bbb]---------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(20), true, - MakeValue(30), true + MakePoint({20}), true, + MakePoint({30}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -468,8 +545,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(20), true, - MakeValue(20), true + MakePoint({20}), true, + MakePoint({20}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -483,14 +560,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----[aaa]---------- //------------[bbb]--- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(25), true, - MakeValue(30), true + MakePoint({25}), true, + MakePoint({30}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -507,14 +584,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----------(aaa]--- //----[bbbbbb]------- auto first = MakeRange( - MakeValue(10), false, - MakeValue(20), true + MakePoint({10}), false, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(10), true + MakePoint({1}), true, + MakePoint({10}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -531,14 +608,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----------[aaa]--- //----[bbbbbb)------- auto first = MakeRange( - MakeValue(10), true, - MakeValue(20), true + MakePoint({10}), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(10), false + MakePoint({1}), true, + MakePoint({10}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -555,14 +632,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-----------(aaa]--- //----[bbbbbb)------- auto first = MakeRange( - MakeValue(10), false, - MakeValue(20), true + MakePoint({10}), false, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(10), false + MakePoint({1}), true, + MakePoint({10}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -579,14 +656,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //---------(aaa]--- //----[bbbbbb]------- auto first = MakeRange( - MakeValue(10), false, - MakeValue(20), true + MakePoint({10}), false, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(15), true + MakePoint({1}), true, + MakePoint({15}), true ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -594,8 +671,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), false, - MakeValue(15), true + MakePoint({10}), false, + MakePoint({15}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -609,14 +686,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //---------(aaa]--- //----[bbbbbb)------- auto first = MakeRange( - MakeValue(10), false, - MakeValue(20), true + MakePoint({10}), false, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(15), false + MakePoint({1}), true, + MakePoint({15}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -624,8 +701,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), false, - MakeValue(15), false + MakePoint({10}), false, + MakePoint({15}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -639,14 +716,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //---------(aaa]--- //----[bbbbbbbb)------- auto first = MakeRange( - MakeValue(10), false, - MakeValue(20), true + MakePoint({10}), false, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -654,8 +731,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), false, - MakeValue(20), false + MakePoint({10}), false, + MakePoint({20}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -669,14 +746,14 @@ Y_UNIT_TEST_SUITE(RangeOps) { //---------(aaa)--- //----[bbbbbbbb)------- auto first = MakeRange( - MakeValue(10), false, - MakeValue(20), false + MakePoint({10}), false, + MakePoint({20}), false ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -684,8 +761,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), false, - MakeValue(20), false + MakePoint({10}), false, + MakePoint({20}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -699,15 +776,15 @@ Y_UNIT_TEST_SUITE(RangeOps) { //(-inf; +inf) //----[bbbbbbbb)------- auto first = MakeRange( - MakeLeftInf(), true, + MakeLeftInf(valueType.size()), true, MakeRightInf(), false ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -715,8 +792,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -730,15 +807,15 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-------[aaaaa +inf) //----[bbbbbbbb)------- auto first = MakeRange( - MakeValue(10), true, + MakePoint({10}), true, MakeRightInf(), false ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -746,8 +823,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), true, - MakeValue(20), false + MakePoint({10}), true, + MakePoint({20}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -761,15 +838,15 @@ Y_UNIT_TEST_SUITE(RangeOps) { //--------(aaaaa +inf) //----[bbb)------- auto first = MakeRange( - MakeValue(10), false, + MakePoint({10}), false, MakeRightInf(), false ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); auto second = MakeRange( - MakeValue(1), true, - MakeValue(10), false + MakePoint({1}), true, + MakePoint({10}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -786,15 +863,15 @@ Y_UNIT_TEST_SUITE(RangeOps) { //-------(aaaaa +inf) //----[bbbbbbbb)------- auto first = MakeRange( - MakeValue(10), false, + MakePoint({10}), false, MakeRightInf(), false ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -802,8 +879,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(10), false, - MakeValue(20), false + MakePoint({10}), false, + MakePoint({20}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -817,15 +894,15 @@ Y_UNIT_TEST_SUITE(RangeOps) { //(-inf; aaa]------ //----[bbbbbbbb)------- auto first = MakeRange( - MakeLeftInf(), true, - MakeValue(10), true + MakeLeftInf(valueType.size()), true, + MakePoint({10}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); auto second = MakeRange( - MakeValue(1), true, - MakeValue(20), false + MakePoint({1}), true, + MakePoint({20}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -833,8 +910,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(1), true, - MakeValue(10), true + MakePoint({1}), true, + MakePoint({10}), true ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); @@ -848,15 +925,15 @@ Y_UNIT_TEST_SUITE(RangeOps) { //(-inf; aaa]------ //----[bbbbbbbb)------- auto first = MakeRange( - MakeLeftInf(), true, - MakeValue(20), true + MakeLeftInf(valueType.size()), true, + MakePoint({20}), true ); Cerr << "first " << DebugPrintRange(valueType, first.ToTableRange(), *typeRegistry); Y_ASSERT(!first.ToTableRange().IsEmptyRange(valueType)); auto second = MakeRange( - MakeValue(1), true, - MakeValue(10), false + MakePoint({1}), true, + MakePoint({10}), false ); Cerr << " second " << DebugPrintRange(valueType, second.ToTableRange(), *typeRegistry); @@ -864,8 +941,8 @@ Y_UNIT_TEST_SUITE(RangeOps) { Cerr << " result " << DebugPrintRange(valueType, range, *typeRegistry); auto correct = MakeRange( - MakeValue(1), true, - MakeValue(10), false + MakePoint({1}), true, + MakePoint({10}), false ); Cerr << " correct " << DebugPrintRange(valueType, correct.ToTableRange(), *typeRegistry); diff --git a/ydb/core/tx/schemeshard/schemeshard__init.cpp b/ydb/core/tx/schemeshard/schemeshard__init.cpp index 03958dedda..20248f6d8f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__init.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__init.cpp @@ -4208,13 +4208,12 @@ struct TSchemeShard::TTxInit : public TTransactionBase<TSchemeShard> { TShardIdx shardIdx = TShardIdx(rowset.GetValue<Schema::IndexBuildShardStatus::OwnerShardIdx>(), rowset.GetValue<Schema::IndexBuildShardStatus::LocalShardIdx>()); - - TIndexBuildInfo::TShardStatus& shardStatus = buildInfo->Shards[shardIdx]; - NKikimrTx::TKeyRange range = rowset.GetValue<Schema::IndexBuildShardStatus::Range>(); - shardStatus.Range.Load(range); + TString lastKeyAck = rowset.GetValue<Schema::IndexBuildShardStatus::LastKeyAck>(); + + buildInfo->Shards.emplace(shardIdx, TIndexBuildInfo::TShardStatus(TSerializedTableRange(range), std::move(lastKeyAck))); + TIndexBuildInfo::TShardStatus& shardStatus = buildInfo->Shards.at(shardIdx); - shardStatus.LastKeyAck = rowset.GetValue<Schema::IndexBuildShardStatus::LastKeyAck>(); shardStatus.Status = rowset.GetValue<Schema::IndexBuildShardStatus::Status>(); shardStatus.DebugMessage = rowset.GetValueOrDefault<Schema::IndexBuildShardStatus::Message>(); diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp index a5fbbae703..50e484da10 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index__progress.cpp @@ -420,12 +420,22 @@ public: return true; } + TSerializedTableRange InfiniteRange(ui32 columns) { + TVector<TCell> vec(columns, TCell()); + TArrayRef<TCell> cells(vec); + return TSerializedTableRange(TSerializedCellVec::Serialize(cells), "", true, false); + } + void InitiateShards(NIceDb::TNiceDb& db, TIndexBuildInfo::TPtr& buildInfo) { TTableInfo::TPtr table = Self->Tables.at(buildInfo->TablePathId); + + auto tableColumns = NTableIndex::ExtractInfo(table); // skip dropped columns + const TSerializedTableRange infiniteRange = InfiniteRange(tableColumns.Keys.size()); + for (const auto& x: table->GetPartitions()) { Y_VERIFY(Self->ShardInfos.contains(x.ShardIdx)); - buildInfo->Shards.emplace(x.ShardIdx, TIndexBuildInfo::TShardStatus()); + buildInfo->Shards.emplace(x.ShardIdx, TIndexBuildInfo::TShardStatus(infiniteRange, "")); Self->PersistBuildIndexUploadInitiate(db, buildInfo, x.ShardIdx); } } @@ -662,7 +672,7 @@ public: } if (record.HasLastKeyAck()) { - { + if (shardStatus.LastKeyAck) { //check that all LastKeyAcks are monotonously increase TTableInfo::TPtr tableInfo = Self->Tables.at(buildInfo->TablePathId); TVector<ui16> keyTypes; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp index b5a9000841..9057806cb8 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.cpp @@ -1727,14 +1727,10 @@ void TImportInfo::AddNotifySubscriber(const TActorId &actorId) { Subscribers.insert(actorId); } -TIndexBuildInfo::TShardStatus::TShardStatus() { - TCell cell; - TVector<TCell> vec = {cell}; - TArrayRef<TCell> cells(vec); - Range = TSerializedTableRange(TSerializedCellVec::Serialize(cells), "", true, false); - - LastKeyAck = Range.From.GetBuffer(); // -inf -} +TIndexBuildInfo::TShardStatus::TShardStatus(TSerializedTableRange range, TString lastKeyAck) + : Range(std::move(range)) + , LastKeyAck(std::move(lastKeyAck)) +{} TColumnFamiliesMerger::TColumnFamiliesMerger(NKikimrSchemeOp::TPartitionConfig &container) : Container(container) diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index cae07292f8..e241dd170b 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -2580,7 +2580,7 @@ struct TIndexBuildInfo: public TSimpleRefCount<TIndexBuildInfo> { TBillingStats Processed; TBillingStats Billed; - TShardStatus(); + TShardStatus(TSerializedTableRange range, TString lastKeyAck); TString ToString(TShardIdx shardIdx = InvalidShardIdx) const { TStringBuilder result; diff --git a/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp b/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp index 543d52895d..cc6c183f61 100644 --- a/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp +++ b/ydb/core/tx/schemeshard/ut_index_build_reboots.cpp @@ -9,6 +9,8 @@ using namespace NSchemeShardUT_Private; static void WriteRows(TTestActorRuntime& runtime, ui64 tabletId, ui32 key, ui32 index) { TString writeQuery = Sprintf(R"( ( + (let keyNull '( '('key (Null) ) ) ) + (let row0 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) (let key0 '( '('key (Uint32 '%u ) ) ) ) (let row0 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) (let key1 '( '('key (Uint32 '%u ) ) ) ) @@ -31,6 +33,7 @@ static void WriteRows(TTestActorRuntime& runtime, ui64 tabletId, ui32 key, ui32 (let row9 '( '('index (Uint32 '%u ) ) '('value (Utf8 'aaaa) ) ) ) (return (AsList + (UpdateRow '__user__Table keyNull row0) (UpdateRow '__user__Table key0 row0) (UpdateRow '__user__Table key1 row1) (UpdateRow '__user__Table key2 row2) @@ -45,6 +48,7 @@ static void WriteRows(TTestActorRuntime& runtime, ui64 tabletId, ui32 key, ui32 ) ) )", + 1000*index + 0, 1000*key + 0, 1000*index + 0, 1000*key + 1, 1000*index + 1, 1000*key + 2, 1000*index + 2, @@ -84,6 +88,26 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { for (ui32 delta = 0; delta < 2; ++delta) { WriteRows(runtime, TTestTxConfig::FakeHiveTablets, 1 + delta, 100 + delta); } + + // Check src shard has the lead key as null + { + NKikimrMiniKQL::TResult result; + TString err; + ui32 status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets, R"( + ( + (let range '('('key (Null) (Void)))) + (let columns '('key 'index)) + (let result (SelectRange '__user__Table range columns '())) + (return (AsList (SetResult 'Result result))) + ) + )", result, err); + + UNIT_ASSERT_VALUES_EQUAL_C(status, static_cast<ui32>(NKikimrProto::OK), err); + UNIT_ASSERT_VALUES_EQUAL(err, ""); + + // V -- here the null key + NKqp::CompareYson(R"([[[[[["101000"];#];[["100000"];["1000"]];[["100001"];["1001"]];[["100002"];["1002"]];[["100003"];["1003"]];[["100004"];["1004"]];[["100005"];["1005"]];[["100006"];["1006"]];[["100007"];["1007"]];[["100008"];["1008"]];[["100009"];["1009"]];[["101000"];["2000"]];[["101001"];["2001"]];[["101002"];["2002"]];[["101003"];["2003"]];[["101004"];["2004"]];[["101005"];["2005"]];[["101006"];["2006"]];[["101007"];["2007"]];[["101008"];["2008"]];[["101009"];["2009"]]];%false]]])", result); + } } AsyncBuilIndex(runtime, ++t.TxId, TTestTxConfig::SchemeShard, "/MyRoot", "/MyRoot/dir/Table", "index1", {"index"}); @@ -120,7 +144,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { TString err; ui32 status = LocalMiniKQL(runtime, TTestTxConfig::FakeHiveTablets+2, R"( ( - (let range '( '('index (Uint32 '0) (Void) ) '('key (Uint32 '0) (Void) ))) + (let range '( '('index (Null) (Void)) '('key (Null) (Void)))) (let columns '('key 'index) ) (let result (SelectRange '__user__indexImplTable range columns '())) (return (AsList (SetResult 'Result result) )) @@ -130,7 +154,8 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::OK, err); UNIT_ASSERT_VALUES_EQUAL(err, ""); - NKqp::CompareYson(R"([[[[[["100000"];["1000"]];[["100001"];["1001"]];[["100002"];["1002"]];[["100003"];["1003"]];[["100004"];["1004"]];[["100005"];["1005"]];[["100006"];["1006"]];[["100007"];["1007"]];[["100008"];["1008"]];[["100009"];["1009"]];[["101000"];["2000"]];[["101001"];["2001"]];[["101002"];["2002"]];[["101003"];["2003"]];[["101004"];["2004"]];[["101005"];["2005"]];[["101006"];["2006"]];[["101007"];["2007"]];[["101008"];["2008"]];[["101009"];["2009"]]];%false]]])", result); + // record with null is there -> V -- here is the null + NKqp::CompareYson(R"([[[[[["100000"];["1000"]];[["100001"];["1001"]];[["100002"];["1002"]];[["100003"];["1003"]];[["100004"];["1004"]];[["100005"];["1005"]];[["100006"];["1006"]];[["100007"];["1007"]];[["100008"];["1008"]];[["100009"];["1009"]];[["101000"];#];[["101000"];["2000"]];[["101001"];["2001"]];[["101002"];["2002"]];[["101003"];["2003"]];[["101004"];["2004"]];[["101005"];["2005"]];[["101006"];["2006"]];[["101007"];["2007"]];[["101008"];["2008"]];[["101009"];["2009"]]];%false]]])", result); } }); } @@ -200,7 +225,7 @@ Y_UNIT_TEST_SUITE(IndexBuildTestReboots) { UNIT_ASSERT_VALUES_EQUAL_C(status, NKikimrProto::OK, err); UNIT_ASSERT_VALUES_EQUAL(err, ""); - NKqp::CompareYson(R"([[[[[["100000"];["1000"];["aaaa"]];[["100001"];["1001"];["aaaa"]];[["100002"];["1002"];["aaaa"]];[["100003"];["1003"];["aaaa"]];[["100004"];["1004"];["aaaa"]];[["100005"];["1005"];["aaaa"]];[["100006"];["1006"];["aaaa"]];[["100007"];["1007"];["aaaa"]];[["100008"];["1008"];["aaaa"]];[["100009"];["1009"];["aaaa"]];[["101000"];["2000"];["aaaa"]];[["101001"];["2001"];["aaaa"]];[["101002"];["2002"];["aaaa"]];[["101003"];["2003"];["aaaa"]];[["101004"];["2004"];["aaaa"]];[["101005"];["2005"];["aaaa"]];[["101006"];["2006"];["aaaa"]];[["101007"];["2007"];["aaaa"]];[["101008"];["2008"];["aaaa"]];[["101009"];["2009"];["aaaa"]]];%false]]])", result); + NKqp::CompareYson(R"([[[[[["100000"];["1000"];["aaaa"]];[["100001"];["1001"];["aaaa"]];[["100002"];["1002"];["aaaa"]];[["100003"];["1003"];["aaaa"]];[["100004"];["1004"];["aaaa"]];[["100005"];["1005"];["aaaa"]];[["100006"];["1006"];["aaaa"]];[["100007"];["1007"];["aaaa"]];[["100008"];["1008"];["aaaa"]];[["100009"];["1009"];["aaaa"]];[["101000"];#;["aaaa"]];[["101000"];["2000"];["aaaa"]];[["101001"];["2001"];["aaaa"]];[["101002"];["2002"];["aaaa"]];[["101003"];["2003"];["aaaa"]];[["101004"];["2004"];["aaaa"]];[["101005"];["2005"];["aaaa"]];[["101006"];["2006"];["aaaa"]];[["101007"];["2007"];["aaaa"]];[["101008"];["2008"];["aaaa"]];[["101009"];["2009"];["aaaa"]]];%false]]])", result); } }); } |