diff options
author | dcherednik <dcherednik@ydb.tech> | 2023-08-30 16:08:24 +0300 |
---|---|---|
committer | dcherednik <dcherednik@ydb.tech> | 2023-08-30 16:29:10 +0300 |
commit | 16ecdc10e833b495182ee6ae1ddb7b6384ddd308 (patch) | |
tree | 78cbeb190e9a83e6f534415a205bfcbdba53f5f5 | |
parent | f54e6b6e47c6bb1e42c0870eabc8e893f5217f83 (diff) | |
download | ydb-16ecdc10e833b495182ee6ae1ddb7b6384ddd308.tar.gz |
Unique constraint support in kqp optimizer for INSERT query. KIKIMR-19064
24 files changed, 791 insertions, 205 deletions
diff --git a/ydb/core/kqp/host/kqp_gateway_proxy.cpp b/ydb/core/kqp/host/kqp_gateway_proxy.cpp index 962c82ea2f..69f74faccc 100644 --- a/ydb/core/kqp/host/kqp_gateway_proxy.cpp +++ b/ydb/core/kqp/host/kqp_gateway_proxy.cpp @@ -460,6 +460,9 @@ public: case NYql::TIndexDescription::EType::GlobalAsync: indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync); break; + case NYql::TIndexDescription::EType::GlobalSyncUnique: + indexDesc->SetType(NKikimrSchemeOp::EIndexType::EIndexTypeGlobalUnique); + break; } indexDesc->SetState(static_cast<::NKikimrSchemeOp::EIndexState>(index.State)); diff --git a/ydb/core/kqp/host/kqp_type_ann.cpp b/ydb/core/kqp/host/kqp_type_ann.cpp index b29e6838a4..2d9174ddb5 100644 --- a/ydb/core/kqp/host/kqp_type_ann.cpp +++ b/ydb/core/kqp/host/kqp_type_ann.cpp @@ -454,13 +454,18 @@ TStatus AnnotateLookupTable(const TExprNode::TPtr& node, TExprContext& ctx, cons } } + auto tableDbg = [&]() { + return TStringBuilder() << "Lookup: " << structType->ToString() + << ", for table: " << table.second->Metadata->Name; + }; + if (!keyColumnsCount) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup has no key columns.")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup has no key columns. " + tableDbg())); return TStatus::Error; } if (structType->GetSize() != keyColumnsCount) { - ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup contains non-key columns.")); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Table lookup contains non-key columns. " + tableDbg())); return TStatus::Error; } diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects.cpp index 0e8dc15496..ab36056a9e 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects.cpp @@ -150,7 +150,7 @@ TMaybe<TCondenseInputResult> CondenseInputToDictByPk(const TExprBase& input, con .Args({"row_list"}) .Body<TCoToDict>() .List("row_list") - .KeySelector(MakeTableKeySelector(table, input.Pos(), ctx)) + .KeySelector(MakeTableKeySelector(table.Metadata, input.Pos(), ctx)) .PayloadSelector(payloadSelector) .Settings() .Add().Build("One") @@ -167,11 +167,39 @@ TMaybe<TCondenseInputResult> CondenseInputToDictByPk(const TExprBase& input, con }; } -TCoLambda MakeTableKeySelector(const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx) { +TCoLambda MakeTableKeySelector(const TKikimrTableMetadataPtr meta, TPositionHandle pos, TExprContext& ctx) { auto keySelectorArg = TCoArgument(ctx.NewArgument(pos, "key_selector")); TVector<TExprBase> keyTuples; - for (const auto& key : table.Metadata->KeyColumnNames) { + keyTuples.reserve(meta->KeyColumnNames.size()); + for (const auto& key : meta->KeyColumnNames) { + auto tuple = Build<TCoNameValueTuple>(ctx, pos) + .Name().Build(key) + .Value<TCoMember>() + .Struct(keySelectorArg) + .Name().Build(key) + .Build() + .Done(); + + keyTuples.emplace_back(tuple); + } + + return Build<TCoLambda>(ctx, pos) + .Args({keySelectorArg}) + .Body<TCoAsStruct>() + .Add(keyTuples) + .Build() + .Done(); +} + +NYql::NNodes::TCoLambda MakeIndexPrefixKeySelector(const NYql::TIndexDescription& indexDesc, NYql::TPositionHandle pos, + NYql::TExprContext& ctx) +{ + auto keySelectorArg = TCoArgument(ctx.NewArgument(pos, "fk_selector")); + + TVector<TExprBase> keyTuples; + keyTuples.reserve(indexDesc.KeyColumns.size()); + for (const auto& key : indexDesc.KeyColumns) { auto tuple = Build<TCoNameValueTuple>(ctx, pos) .Name().Build(key) .Value<TCoMember>() diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h index 0533acd279..3fc7bbf9d5 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h @@ -35,7 +35,19 @@ NYql::NNodes::TMaybeNode<NYql::NNodes::TDqPhyPrecompute> PrecomputeTableLookupDi const THashSet<TString>& dataColumns, const THashSet<TString>& keyColumns, NYql::TPositionHandle pos, NYql::TExprContext& ctx); -NYql::NNodes::TCoLambda MakeTableKeySelector(const NYql::TKikimrTableDescription& table, NYql::TPositionHandle pos, +// Creates key selector using PK of given table +NYql::NNodes::TCoLambda MakeTableKeySelector(const NYql::TKikimrTableMetadataPtr tableMeta, NYql::TPositionHandle pos, + NYql::TExprContext& ctx); + +// Creates key selector using user provided index columns. +// It is important to note. This function looks at the _user_prvided_ set of columns. +// Example: table with columns a, b, pk(a) +// case 1: +// user creates index for column b, index table pk(b,a). But this fuction must create selector only for column b +// case 2: +// user creates index for columns b, a, index table pk is same pk(b,a). But this function must crete selector for b, a + +NYql::NNodes::TCoLambda MakeIndexPrefixKeySelector(const NYql::TIndexDescription& indexDesc, NYql::TPositionHandle pos, NYql::TExprContext& ctx); NYql::NNodes::TCoLambda MakeRowsPayloadSelector(const NYql::NNodes::TCoAtomList& columns, diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert.cpp index c6782b6c03..975744fdff 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_insert.cpp @@ -7,19 +7,23 @@ using namespace NYql; using namespace NYql::NDq; using namespace NYql::NNodes; -TMaybeNode<TDqCnUnionAll> MakeConditionalInsertRows(const TExprBase& input, const TKikimrTableDescription& table, - bool abortOnError, TPositionHandle pos, TExprContext& ctx) -{ - auto condenseResult = CondenseInput(input, ctx); - if (!condenseResult) { - return {}; - } +namespace { - TCoArgument rowsListArg(ctx.NewArgument(pos, "rows_list")); +struct TUniqCheckNodes { + using TIndexId = int; + static constexpr TIndexId INVALID_INDEX_ID = -1; + TExprNode::TPtr DictKeys; + TExprNode::TPtr UniqCmp; + TIndexId IndexId = INVALID_INDEX_ID; +}; +TUniqCheckNodes MakeUniqCheckNodes(const TCoLambda& selector, + const TExprBase& rowsListArg, TPositionHandle pos, TExprContext& ctx) +{ + TUniqCheckNodes result; auto dict = Build<TCoToDict>(ctx, pos) .List(rowsListArg) - .KeySelector(MakeTableKeySelector(table, pos, ctx)) + .KeySelector(selector) .PayloadSelector() .Args({"stub"}) .Body<TCoVoid>() @@ -29,109 +33,47 @@ TMaybeNode<TDqCnUnionAll> MakeConditionalInsertRows(const TExprBase& input, cons .Add().Build("One") .Add().Build("Hashed") .Build() - .Done(); + .Done().Ptr(); - auto dictKeys = Build<TCoDictKeys>(ctx, pos) + result.DictKeys = Build<TCoDictKeys>(ctx, pos) .Dict(dict) - .Done(); + .Done().Ptr(); - auto areKeysUniqueValue = Build<TCoCmpEqual>(ctx, pos) + result.UniqCmp = Build<TCoCmpEqual>(ctx, pos) .Left<TCoLength>() .List(rowsListArg) .Build() .Right<TCoLength>() .List(dict) .Build() - .Done(); - - auto variantType = Build<TCoVariantType>(ctx, pos) - .UnderlyingType<TCoTupleType>() - .Add<TCoTypeOf>() - .Value(rowsListArg) - .Build() - .Add<TCoTypeOf>() - .Value(dictKeys) - .Build() - .Add<TCoTypeOf>() - .Value(areKeysUniqueValue) - .Build() - .Build() - .Done(); - - auto computeKeysStage = Build<TDqStage>(ctx, pos) - .Inputs() - .Add(condenseResult->StageInputs) - .Build() - .Program() - .Args(condenseResult->StageArgs) - .Body<TCoFlatMap>() - .Input(condenseResult->Stream) - .Lambda() - .Args({rowsListArg}) - .Body<TCoAsList>() - .Add<TCoVariant>() - .Item(rowsListArg) - .Index().Build("0") - .VarType(variantType) - .Build() - .Add<TCoVariant>() - .Item(dictKeys) - .Index().Build("1") - .VarType(variantType) - .Build() - .Add<TCoVariant>() - .Item(areKeysUniqueValue) - .Index().Build("2") - .VarType(variantType) - .Build() - .Build() - .Build() - .Build() - .Build() - .Settings().Build() - .Done(); + .Done().Ptr(); - auto inputPrecompute = Build<TDqPhyPrecompute>(ctx, pos) - .Connection<TDqCnValue>() - .Output() - .Stage(computeKeysStage) - .Index().Build("0") - .Build() - .Build() - .Done(); + return result; +} +TDqCnUnionAll CreateLookupStageWithConnection(const TDqStage& computeKeysStage, size_t index, const NYql::TKikimrTableMetadata& meta, + TExprNode::TPtr _false, TPositionHandle pos, TExprContext& ctx) +{ auto lookupKeysPrecompute = Build<TDqPhyPrecompute>(ctx, pos) .Connection<TDqCnValue>() .Output() .Stage(computeKeysStage) - .Index().Build("1") + .Index().Build(IntToString<10>(index)) .Build() .Build() .Done(); - auto areKeysUniquePrecompute = Build<TDqPhyPrecompute>(ctx, pos) - .Connection<TDqCnValue>() - .Output() - .Stage(computeKeysStage) - .Index().Build("2") - .Build() - .Build() - .Done(); - - auto _true = MakeBool(pos, true, ctx); - auto _false = MakeBool(pos, false, ctx); - // Returns optional<bool>: <none> - nothing found, <false> - at least one key exists - auto lookupStage = Build<TDqStage>(ctx, pos) + auto stage = Build<TDqStage>(ctx, pos) .Inputs() .Add(lookupKeysPrecompute) .Build() .Program() .Args({"keys_list"}) - .Body<TCoFlatMap>() + .Body<TCoMap>() .Input<TCoTake>() .Input<TKqpLookupTable>() - .Table(BuildTableMeta(table, pos, ctx)) + .Table(BuildTableMeta(meta, pos, ctx)) .LookupKeys<TCoIterator>() .List("keys_list") .Build() @@ -153,33 +95,261 @@ TMaybeNode<TDqCnUnionAll> MakeConditionalInsertRows(const TExprBase& input, cons .Settings().Build() .Done(); - // Returns <bool>: <true> - nothing found, <false> - at least one key exists - auto aggrStage = Build<TDqStage>(ctx, pos) - .Inputs() - .Add<TDqCnUnionAll>() - .Output() - .Stage(lookupStage) - .Index().Build("0") - .Build() - .Build() + return Build<TDqCnUnionAll>(ctx, pos) + .Output() + .Stage(stage) + .Index().Build("0") .Build() - .Program() - .Args({"row"}) - .Body<TCoCondense>() - .Input("row") - .State(_true) - .SwitchHandler() - .Args({"item", "state"}) - .Body(_false) - .Build() - .UpdateHandler() - .Args({"item", "state"}) - .Body(_false) + .Done(); +} + +class TUniqBuildHelper { + static TVector<TUniqCheckNodes> Prepare(const TCoArgument& rowsListArg, const TKikimrTableDescription& table, + TPositionHandle pos, TExprContext& ctx) + { + + TVector<TUniqCheckNodes> checks; + checks.emplace_back(MakeUniqCheckNodes(MakeTableKeySelector(table.Metadata, pos, ctx), rowsListArg, pos, ctx)); + + // make uniq check for each uniq constraint + for (size_t i = 0; i < table.Metadata->Indexes.size(); i++) { + if (table.Metadata->Indexes[i].State != TIndexDescription::EIndexState::Ready) + continue; + if (table.Metadata->Indexes[i].Type != TIndexDescription::EType::GlobalSyncUnique) + continue; + + // Compatibility with PG semantic - allow multiple null in columns with unique constaint + TVector<TCoAtom> skipNullColumns; + skipNullColumns.reserve(table.Metadata->Indexes[i].KeyColumns.size()); + for (const auto& column : table.Metadata->Indexes[i].KeyColumns) { + TCoAtom atom(ctx.NewAtom(pos, column)); + skipNullColumns.emplace_back(atom); + } + + auto skipNull = Build<TCoSkipNullMembers>(ctx, pos) + .Input(rowsListArg) + .Members().Add(skipNullColumns).Build() + .Done(); + + checks.emplace_back(MakeUniqCheckNodes(MakeIndexPrefixKeySelector(table.Metadata->Indexes[i], pos, ctx), skipNull, pos, ctx)); + YQL_ENSURE(i < Max<TUniqCheckNodes::TIndexId>()); + checks.back().IndexId = i; + } + return checks; + } + +public: + TUniqBuildHelper(const TKikimrTableDescription& table, TPositionHandle pos, TExprContext& ctx) + : RowsListArg(ctx.NewArgument(pos, "rows_list")) + , Checks(Prepare(RowsListArg, table, pos, ctx)) + {} + + size_t GetChecksNum() const { + return Checks.size(); + } + + TDqStage CreateComputeKeysStage(const TCondenseInputResult& condenseResult, TPositionHandle pos, TExprContext& ctx) const + { + // Number of items for output list 2 for each table + 1 for params itself + const size_t nItems = Checks.size() * 2 + 1; + TVector<TExprBase> types; + types.reserve(nItems); + + types.emplace_back( + Build<TCoTypeOf>(ctx, pos) + .Value(RowsListArg) + .Done() + ); + + for (size_t i = 0; i < Checks.size(); i++) { + types.emplace_back( + Build<TCoTypeOf>(ctx, pos) + .Value(Checks[i].DictKeys) + .Done() + ); + types.emplace_back( + Build<TCoTypeOf>(ctx, pos) + .Value(Checks[i].UniqCmp) + .Done() + ); + } + + auto variantType = Build<TCoVariantType>(ctx, pos) + .UnderlyingType<TCoTupleType>() + .Add(types) + .Build() + .Done(); + + TVector<TExprBase> variants; + variants.reserve(nItems); + + variants.emplace_back( + Build<TCoVariant>(ctx, pos) + .Item(RowsListArg) + .Index().Build("0") + .VarType(variantType) + .Done() + ); + + for (size_t i = 0, ch = 1; i < Checks.size(); i++) { + variants.emplace_back( + Build<TCoVariant>(ctx, pos) + .Item(Checks[i].DictKeys) + .Index().Build(IntToString<10>(ch++)) + .VarType(variantType) + .Done() + ); + variants.emplace_back( + Build<TCoVariant>(ctx, pos) + .Item(Checks[i].UniqCmp) + .Index().Build(IntToString<10>(ch++)) + .VarType(variantType) + .Done() + ); + } + + return Build<TDqStage>(ctx, pos) + .Inputs() + .Add(condenseResult.StageInputs) + .Build() + .Program() + .Args(condenseResult.StageArgs) + .Body<TCoFlatMap>() + .Input(condenseResult.Stream) + .Lambda() + .Args({RowsListArg}) + .Body<TCoAsList>() + .Add(variants) + .Build() + .Build() .Build() .Build() - .Build() - .Settings().Build() + .Settings().Build() + .Done(); + } + + TDqPhyPrecompute CreateInputPrecompute(const TDqStage& computeKeysStage, TPositionHandle pos, TExprContext& ctx) { + return Build<TDqPhyPrecompute>(ctx, pos) + .Connection<TDqCnValue>() + .Output() + .Stage(computeKeysStage) + .Index().Build("0") + .Build() + .Build() .Done(); + } + + TVector<TExprBase> CreateUniquePrecompute(const TDqStage& computeKeysStage, TPositionHandle pos, TExprContext& ctx) { + TVector<TExprBase> uniquePrecomputes; + uniquePrecomputes.reserve(Checks.size()); + for (size_t i = 0, output_index = 2; i < Checks.size(); i++, output_index += 2) { + uniquePrecomputes.emplace_back(Build<TDqPhyPrecompute>(ctx, pos) + .Connection<TDqCnValue>() + .Output() + .Stage(computeKeysStage) + .Index().Build(IntToString<10>(output_index)) + .Build() + .Build() + .Done() + ); + } + return uniquePrecomputes; + } + + struct TLookupNodes { + TLookupNodes(size_t sz) { + Stages.reserve(sz); + Args.reserve(sz); + } + TVector<TExprBase> Stages; + TVector<TCoArgument> Args; + }; + + TDqStage CreateLookupExistStage(const TDqStage& computeKeysStage, const TKikimrTableDescription& table, + TExprNode::TPtr _true, TPositionHandle pos, TExprContext& ctx) + { + TLookupNodes lookupNodes(Checks.size()); + + auto _false = MakeBool(pos, false, ctx); + + lookupNodes.Stages.emplace_back( + // 1 is id of precompute key stage output for primary key + CreateLookupStageWithConnection(computeKeysStage, 1, *table.Metadata, _false, pos, ctx) + ); + lookupNodes.Args.emplace_back( + Build<TCoArgument>(ctx, pos) + .Name("arg0") + .Done() + ); + + // For each index create lookup stage using computeKeysStage. + // 3 is id of of precompute key stage output for index + for (size_t i = 1, stage_out = 3; i < Checks.size(); i++, stage_out += 2) { + const auto indexId = Checks[i].IndexId; + YQL_ENSURE(indexId >=0); + YQL_ENSURE((size_t)indexId < table.Metadata->SecondaryGlobalIndexMetadata.size()); + lookupNodes.Stages.emplace_back( + CreateLookupStageWithConnection(computeKeysStage, stage_out, + *(table.Metadata->SecondaryGlobalIndexMetadata[indexId]), _false, pos, ctx) + ); + lookupNodes.Args.emplace_back( + Build<TCoArgument>(ctx, pos) + .Name(TString("arg") + IntToString<10>(i)) + .Done() + ); + } + + return Build<TDqStage>(ctx, pos) + .Inputs() + .Add(lookupNodes.Stages) + .Build() + .Program() + .Args(lookupNodes.Args) + .Body<TCoCondense>() + .Input<TCoToStream>() + .Input<TCoExtend>() + .Add(TVector<TExprBase>(lookupNodes.Args.begin(), lookupNodes.Args.end())) + .Build() + .Build() + .State(_true) + .SwitchHandler() + .Args({"item", "state"}) + .Body(_false) + .Build() + .UpdateHandler() + .Args({"item", "state"}) + .Body(_false) + .Build() + .Build() + .Build() + .Settings().Build() + .Done(); + } + +private: + TCoArgument RowsListArg; + TVector<TUniqCheckNodes> Checks; +}; + +} + +TMaybeNode<TDqCnUnionAll> MakeConditionalInsertRows(const TExprBase& input, const TKikimrTableDescription& table, + bool abortOnError, TPositionHandle pos, TExprContext& ctx) +{ + auto condenseResult = CondenseInput(input, ctx); + if (!condenseResult) { + return {}; + } + + TUniqBuildHelper helper(table, pos, ctx); + auto computeKeysStage = helper.CreateComputeKeysStage(condenseResult.GetRef(), pos, ctx); + + auto inputPrecompute = helper.CreateInputPrecompute(computeKeysStage, pos, ctx); + auto uniquePrecomputes = helper.CreateUniquePrecompute(computeKeysStage, pos, ctx); + + auto _true = MakeBool(pos, true, ctx); + + auto aggrStage = helper.CreateLookupExistStage(computeKeysStage, table, _true, pos, ctx); // Returns <bool>: <true> - no existing keys, <false> - at least one key exists auto noExistingKeysPrecompute = Build<TDqPhyPrecompute>(ctx, pos) @@ -191,21 +361,30 @@ TMaybeNode<TDqCnUnionAll> MakeConditionalInsertRows(const TExprBase& input, cons .Build() .Done(); - // Build condition checks depending on INSERT kind - TCoArgument inputRowsArg(ctx.NewArgument(pos, "input_rows")); - TCoArgument areKeysUniquesArg(ctx.NewArgument(pos, "are_keys_unique")); - TCoArgument noExistingKeysArg(ctx.NewArgument(pos, "no_existing_keys")); + struct TUniqueCheckNodes { + TUniqueCheckNodes(size_t sz) { + Bodies.reserve(sz); + Args.reserve(sz); + } + TVector<TExprNode::TPtr> Bodies; + TVector<TCoArgument> Args; + } uniqueCheckNodes(helper.GetChecksNum()); - TExprNode::TPtr uniqueKeysCheck; + TCoArgument noExistingKeysArg(ctx.NewArgument(pos, "no_existing_keys")); TExprNode::TPtr noExistingKeysCheck; + // Build condition checks depending on INSERT kind if (abortOnError) { - uniqueKeysCheck = Build<TKqpEnsure>(ctx, pos) - .Value(_true) - .Predicate(areKeysUniquesArg) - .IssueCode().Build(ToString((ui32) TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION)) - .Message(MakeMessage("Duplicated keys found.", pos, ctx)) - .Done().Ptr(); + for (size_t i = 0; i < helper.GetChecksNum(); i++) { + uniqueCheckNodes.Args.emplace_back(ctx.NewArgument(pos, "are_keys_unique")); + uniqueCheckNodes.Bodies.emplace_back(Build<TKqpEnsure>(ctx, pos) + .Value(_true) + .Predicate(uniqueCheckNodes.Args.back()) + .IssueCode().Build(ToString((ui32) TIssuesIds::KIKIMR_CONSTRAINT_VIOLATION)) + .Message(MakeMessage("Duplicated keys found.", pos, ctx)) + .Done().Ptr() + ); + } noExistingKeysCheck = Build<TKqpEnsure>(ctx, pos) .Value(_true) @@ -214,25 +393,36 @@ TMaybeNode<TDqCnUnionAll> MakeConditionalInsertRows(const TExprBase& input, cons .Message(MakeMessage("Conflict with existing key.", pos, ctx)) .Done().Ptr(); } else { - uniqueKeysCheck = areKeysUniquesArg.Ptr(); + for (size_t i = 0; i < helper.GetChecksNum(); i++) { + uniqueCheckNodes.Args.emplace_back(ctx.NewArgument(pos, "are_keys_unique")); + uniqueCheckNodes.Bodies.emplace_back(uniqueCheckNodes.Args.back().Ptr()); + } + noExistingKeysCheck = noExistingKeysArg.Ptr(); } + + TCoArgument inputRowsArg(ctx.NewArgument(pos, "input_rows")); // Final pure compute stage to compute rows to insert and check for both conditions: // 1. No rows were found in table PK from input // 2. No duplicate PKs were found in input + TVector<TCoArgument> args; + args.reserve(uniqueCheckNodes.Args.size() + 2); + args.emplace_back(inputRowsArg); + args.insert(args.end(), uniqueCheckNodes.Args.begin(), uniqueCheckNodes.Args.end()); + args.emplace_back(noExistingKeysArg); auto conditionStage = Build<TDqStage>(ctx, pos) .Inputs() .Add(inputPrecompute) - .Add(areKeysUniquePrecompute) + .Add(uniquePrecomputes) .Add(noExistingKeysPrecompute) .Build() .Program() - .Args({inputRowsArg, areKeysUniquesArg, noExistingKeysArg}) + .Args(args) .Body<TCoToStream>() .Input<TCoIfStrict>() .Predicate<TCoAnd>() - .Add(uniqueKeysCheck) + .Add(uniqueCheckNodes.Bodies) .Add(noExistingKeysCheck) .Build() .ThenValue(inputRowsArg) diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp index 53358a1954..58de519dfd 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp @@ -23,7 +23,7 @@ TRowsAndKeysResult PrecomputeRowsAndKeys(const TCondenseInputResult& condenseRes auto extractKeys = Build<TCoMap>(ctx, pos) .Input(rowsListArg) - .Lambda(MakeTableKeySelector(table, pos, ctx)) + .Lambda(MakeTableKeySelector(table.Metadata, pos, ctx)) .Done(); auto variantType = Build<TCoVariantType>(ctx, pos) diff --git a/ydb/core/kqp/provider/yql_kikimr_gateway.h b/ydb/core/kqp/provider/yql_kikimr_gateway.h index 8af6d91042..37f0191900 100644 --- a/ydb/core/kqp/provider/yql_kikimr_gateway.h +++ b/ydb/core/kqp/provider/yql_kikimr_gateway.h @@ -49,7 +49,7 @@ struct TIndexDescription { enum class EType : ui32 { GlobalSync = 0, GlobalAsync = 1, - + GlobalSyncUnique = 2, }; // Index states here must be in sync with NKikimrSchemeOp::EIndexState protobuf @@ -107,6 +107,8 @@ struct TIndexDescription { auto type = NYql::TIndexDescription::EType::GlobalSync; if (index.GetType() == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync) { type = NYql::TIndexDescription::EType::GlobalAsync; + } else if (index.GetType() == NKikimrSchemeOp::EIndexType::EIndexTypeGlobalUnique) { + type = NYql::TIndexDescription::EType::GlobalSyncUnique; } return type; @@ -140,6 +142,8 @@ struct TIndexDescription { switch (Type) { case EType::GlobalSync: return true; + case EType::GlobalSyncUnique: + return true; case EType::GlobalAsync: return false; } diff --git a/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp b/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp index 0993ef4c28..fb97a0035f 100644 --- a/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp +++ b/ydb/core/kqp/ut/indexes/kqp_indexes_multishard_ut.cpp @@ -2,6 +2,7 @@ #include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h> #include <ydb/public/sdk/cpp/client/ydb_table/table.h> +#include <ydb/core/protos/flat_scheme_op.pb.h> #include <library/cpp/json/json_reader.h> #include <library/cpp/threading/local_executor/local_executor.h> @@ -17,6 +18,10 @@ using namespace NYdb::NScripting; namespace { +const NKikimrSchemeOp::EIndexType IG_ASYNC = NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync; +const NKikimrSchemeOp::EIndexType IG_SYNC = NKikimrSchemeOp::EIndexType::EIndexTypeGlobal; +const NKikimrSchemeOp::EIndexType IG_UNIQUE = NKikimrSchemeOp::EIndexType::EIndexTypeGlobalUnique; + NYdb::NTable::TDataQueryResult ExecuteDataQuery(TSession& session, const TString& query) { const auto txSettings = TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx(); return session.ExecuteDataQuery(query, txSettings, @@ -29,7 +34,7 @@ NYdb::NTable::TDataQueryResult ExecuteDataQuery(TSession& session, const TString TExecDataQuerySettings().KeepInQueryCache(true)).ExtractValueSync(); } -void CreateTableWithMultishardIndex(Tests::TClient& client, bool async) { +void CreateTableWithMultishardIndex(Tests::TClient& client, NKikimrSchemeOp::EIndexType type) { const TString scheme = R"(Name: "MultiShardIndexed" Columns { Name: "key" Type: "Uint64" } Columns { Name: "fk" Type: "Uint32" } @@ -42,14 +47,49 @@ void CreateTableWithMultishardIndex(Tests::TClient& client, bool async) { NKikimrSchemeOp::TTableDescription desc; bool parseOk = ::google::protobuf::TextFormat::ParseFromString(scheme, &desc); UNIT_ASSERT(parseOk); - NKikimrSchemeOp::EIndexType type = async - ? NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync - : NKikimrSchemeOp::EIndexType::EIndexTypeGlobal; auto status = client.TClient::CreateTableWithUniformShardedIndex("/Root", desc, "index", {"fk"}, type); UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::MSTATUS_OK); } +void CreateTableWithMultishardIndexComplexFk(Tests::TClient& client, NKikimrSchemeOp::EIndexType type) { + const TString scheme = R"(Name: "MultiShardIndexedComplexFk" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "fk1" Type: "Uint32" } + Columns { Name: "fk2" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 3 } } } } + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } } + )"; + + NKikimrSchemeOp::TTableDescription desc; + bool parseOk = ::google::protobuf::TextFormat::ParseFromString(scheme, &desc); + UNIT_ASSERT(parseOk); + + auto status = client.TClient::CreateTableWithUniformShardedIndex("/Root", desc, "index", {"fk1", "fk2"}, type); + UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::MSTATUS_OK); +} + +void CreateTableWithMultishardIndexComplexFkPk(Tests::TClient& client, NKikimrSchemeOp::EIndexType type) { + const TString scheme = R"(Name: "MultiShardIndexed" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "fk" Type: "Uint32" } + Columns { Name: "value" Type: "Utf8" } + KeyColumnNames: ["key"] + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 3 } } } } + SplitBoundary { KeyPrefix { Tuple { Optional { Uint64: 100 } } } } + )"; + + NKikimrSchemeOp::TTableDescription desc; + bool parseOk = ::google::protobuf::TextFormat::ParseFromString(scheme, &desc); + UNIT_ASSERT(parseOk); + + auto status = client.TClient::CreateTableWithUniformShardedIndex("/Root", desc, "index", {"fk", "key"}, type); + UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::MSTATUS_OK); +} + + void CreateTableWithMultishardIndexAndDataColumn(Tests::TClient& client) { const TString scheme = R"(Name: "MultiShardIndexedWithDataColumn" Columns { Name: "key" Type: "Uint64" } @@ -127,10 +167,265 @@ void FillTable(NYdb::NTable::TSession& session) { } +Y_UNIT_TEST_SUITE(KqpUniqueIndex) { + Y_UNIT_TEST(InsertFkAlreadyExist) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + FillTable(session); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173915, 1000000000, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(InsertFkPkOverlap) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndexComplexFkPk(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + FillTable(session); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173915, 1000000000, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + } + + + Y_UNIT_TEST(InsertNullInPk) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + FillTable(session); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (NULL, 1000000001, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (NULL, 1000000002, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); + } + } + + Y_UNIT_TEST(InsertNullInFk) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + FillTable(session); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173915, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173916, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto yson = ReadTableToYson(session, "/Root/MultiShardIndexed/index/indexImplTable"); + const TString expected = R"([[#;[1173915u]];[#;[1173916u]];[[1000000000u];[1u]];[[2000000000u];[2u]];[[3000000000u];[3u]];[[4294967295u];[4u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST(InsertNullInComplexFk) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndexComplexFk(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexedComplexFk` (key, fk1, fk2, value) VALUES + (1173915, NULL, 1, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexedComplexFk` (key, fk1, fk2, value) VALUES + (1173916, NULL, 1, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexedComplexFk` (key, fk1, fk2, value) VALUES + (1173917, 1, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexedComplexFk` (key, fk1, fk2, value) VALUES + (1173918, 1, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto yson = ReadTableToYson(session, "/Root/MultiShardIndexedComplexFk/index/indexImplTable"); + const TString expected = R"([[#;[1u];[1173915u]];[#;[1u];[1173916u]];[[1u];#;[1173917u]];[[1u];#;[1173918u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST(InsertNullInComplexFkDuplicate) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndexComplexFk(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexedComplexFk` (key, fk1, fk2, value) VALUES + (1173915, NULL, 1, "v1"), + (1173916, NULL, 1, "v1"), + (1173917, 1, NULL, "v1"), + (1173918, 1, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto yson = ReadTableToYson(session, "/Root/MultiShardIndexedComplexFk/index/indexImplTable"); + const TString expected = R"([[#;[1u];[1173915u]];[#;[1u];[1173916u]];[[1u];#;[1173917u]];[[1u];#;[1173918u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST(InsertFkDuplicate) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + FillTable(session); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173915, 1230000001, "v1"), + (1173916, 1230000001, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::PRECONDITION_FAILED, result.GetIssues().ToString()); + } + + { + //Multiple NULL allowed + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173915, NULL, "v1"), + (1173916, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto yson = ReadTableToYson(session, "/Root/MultiShardIndexed/index/indexImplTable"); + const TString expected = R"([[#;[1173915u]];[#;[1173916u]];[[1000000000u];[1u]];[[2000000000u];[2u]];[[3000000000u];[3u]];[[4294967295u];[4u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + } + + Y_UNIT_TEST(InsertComplexFkPkOverlapDuplicate) { + TKikimrRunner kikimr(SyntaxV1Settings()); + CreateTableWithMultishardIndexComplexFkPk(kikimr.GetTestClient(), IG_UNIQUE); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + FillTable(session); + + { + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173915, 1230000001, "v1"), + (1173916, 1230000001, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + //Multiple NULL allowed + const TString query(Q_(R"( + INSERT INTO `/Root/MultiShardIndexed` (key, fk, value) VALUES + (1173917, NULL, "v1"), + (1173918, NULL, "v1"); + )")); + + auto result = ExecuteDataQuery(session, query); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + { + const auto yson = ReadTableToYson(session, "/Root/MultiShardIndexed/index/indexImplTable"); + const TString expected = R"([[#;[1173917u]];[#;[1173918u]];[[1000000000u];[1u]];[[1230000001u];[1173915u]];[[1230000001u];[1173916u]];[[2000000000u];[2u]];[[3000000000u];[3u]];[[4294967295u];[4u]]])"; + UNIT_ASSERT_VALUES_EQUAL(yson, expected); + } + + } +} + Y_UNIT_TEST_SUITE(KqpMultishardIndex) { Y_UNIT_TEST(SortedRangeReadDesc) { TKikimrRunner kikimr(SyntaxV1Settings()); - CreateTableWithMultishardIndex(kikimr.GetTestClient(), false); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_SYNC); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); FillTable(session); @@ -148,7 +443,7 @@ Y_UNIT_TEST_SUITE(KqpMultishardIndex) { Y_UNIT_TEST(SecondaryIndexSelect) { TKikimrRunner kikimr(SyntaxV1Settings()); - CreateTableWithMultishardIndex(kikimr.GetTestClient(), false); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_SYNC); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); FillTable(session); @@ -290,7 +585,7 @@ Y_UNIT_TEST_SUITE(KqpMultishardIndex) { Y_UNIT_TEST(YqWorksFineAfterAlterIndexTableDirectly) { TKikimrRunner kikimr(SyntaxV1Settings()); - CreateTableWithMultishardIndex(kikimr.GetTestClient(), false); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_SYNC); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); @@ -1061,7 +1356,7 @@ Y_UNIT_TEST_SUITE(KqpMultishardIndex) { TKikimrRunner kikimr(serverSettings); auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateTableWithMultishardIndex(kikimr.GetTestClient(), false); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), IG_SYNC); FillTable(session); AssertSuccessResult(session.ExecuteDataQuery(Q1_(R"( @@ -1148,7 +1443,7 @@ Y_UNIT_TEST_SUITE(KqpMultishardIndex) { auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); - CreateTableWithMultishardIndex(kikimr.GetTestClient(), asyncIndex); + CreateTableWithMultishardIndex(kikimr.GetTestClient(), asyncIndex ? IG_ASYNC : IG_SYNC); auto buildParam = [&db](ui64 id) { return db.GetParamsBuilder() diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 9f87bab80e..e534803dc9 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -787,6 +787,7 @@ enum EIndexType { EIndexTypeInvalid = 0; EIndexTypeGlobal = 1; EIndexTypeGlobalAsync = 2; + EIndexTypeGlobalUnique = 3; } enum EIndexState { diff --git a/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt index b780ec496b..5d11701a50 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.darwin-x86_64.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(ut_async_index) add_subdirectory(ut_backup) add_subdirectory(ut_base) add_subdirectory(ut_base_reboots) @@ -25,6 +24,7 @@ add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) add_subdirectory(ut_helpers) +add_subdirectory(ut_index) add_subdirectory(ut_index_build) add_subdirectory(ut_index_build_reboots) add_subdirectory(ut_login) diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt index baab111b28..4d681ae9a5 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux-aarch64.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(ut_async_index) add_subdirectory(ut_backup) add_subdirectory(ut_base) add_subdirectory(ut_base_reboots) @@ -25,6 +24,7 @@ add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) add_subdirectory(ut_helpers) +add_subdirectory(ut_index) add_subdirectory(ut_index_build) add_subdirectory(ut_index_build_reboots) add_subdirectory(ut_login) diff --git a/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt b/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt index baab111b28..4d681ae9a5 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.linux-x86_64.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(ut_async_index) add_subdirectory(ut_backup) add_subdirectory(ut_base) add_subdirectory(ut_base_reboots) @@ -25,6 +24,7 @@ add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) add_subdirectory(ut_helpers) +add_subdirectory(ut_index) add_subdirectory(ut_index_build) add_subdirectory(ut_index_build_reboots) add_subdirectory(ut_login) diff --git a/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt b/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt index 0f61e67004..677fdfd9e1 100644 --- a/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/schemeshard/CMakeLists.windows-x86_64.txt @@ -6,7 +6,6 @@ # original buildsystem will not be accepted. -add_subdirectory(ut_async_index) add_subdirectory(ut_backup) add_subdirectory(ut_base) add_subdirectory(ut_base_reboots) @@ -25,6 +24,7 @@ add_subdirectory(ut_extsubdomain) add_subdirectory(ut_extsubdomain_reboots) add_subdirectory(ut_filestore_reboots) add_subdirectory(ut_helpers) +add_subdirectory(ut_index) add_subdirectory(ut_index_build) add_subdirectory(ut_index_build_reboots) add_subdirectory(ut_login) diff --git a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp index 5324d4c8ae..8b2f74aa19 100644 --- a/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_build_index_tx_base.cpp @@ -266,6 +266,7 @@ void TSchemeShard::TIndexBuilder::TTxBase::Fill(NKikimrIndexBuilder::TIndexBuild switch (indexInfo->IndexType) { case NKikimrSchemeOp::EIndexType::EIndexTypeGlobal: + case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalUnique: *index.mutable_global_index() = Ydb::Table::GlobalIndex(); break; case NKikimrSchemeOp::EIndexType::EIndexTypeGlobalAsync: diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.darwin-x86_64.txt b/ydb/core/tx/schemeshard/ut_index/CMakeLists.darwin-x86_64.txt index 93cd429163..73b5067b40 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.darwin-x86_64.txt +++ b/ydb/core/tx/schemeshard/ut_index/CMakeLists.darwin-x86_64.txt @@ -7,14 +7,14 @@ -add_executable(ydb-core-tx-schemeshard-ut_async_index) -target_compile_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +add_executable(ydb-core-tx-schemeshard-ut_index) +target_compile_options(ydb-core-tx-schemeshard-ut_index PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_include_directories(ydb-core-tx-schemeshard-ut_index PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard ) -target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC +target_link_libraries(ydb-core-tx-schemeshard-ut_index PUBLIC contrib-libs-cxxsupp yutil library-cpp-cpuid_check @@ -23,28 +23,29 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC core-testlib-default tx-schemeshard-ut_helpers ) -target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_link_options(ydb-core-tx-schemeshard-ut_index PRIVATE -Wl,-platform_version,macos,11.0,11.0 -fPIC -fPIC -framework CoreFoundation ) -target_sources(ydb-core-tx-schemeshard-ut_async_index PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_async_index/ut_async_index.cpp +target_sources(ydb-core-tx-schemeshard-ut_index PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp ) set_property( TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY SPLIT_FACTOR 60 ) add_yunittest( NAME - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_ARG --print-before-suite --print-before-test @@ -54,26 +55,26 @@ add_yunittest( ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY LABELS MEDIUM ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY PROCESSORS 1 ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY TIMEOUT 600 ) -target_allocator(ydb-core-tx-schemeshard-ut_async_index +target_allocator(ydb-core-tx-schemeshard-ut_index system_allocator ) -vcs_info(ydb-core-tx-schemeshard-ut_async_index) +vcs_info(ydb-core-tx-schemeshard-ut_index) diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux-aarch64.txt b/ydb/core/tx/schemeshard/ut_index/CMakeLists.linux-aarch64.txt index b010ce9de6..6db7752a6a 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux-aarch64.txt +++ b/ydb/core/tx/schemeshard/ut_index/CMakeLists.linux-aarch64.txt @@ -7,14 +7,14 @@ -add_executable(ydb-core-tx-schemeshard-ut_async_index) -target_compile_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +add_executable(ydb-core-tx-schemeshard-ut_index) +target_compile_options(ydb-core-tx-schemeshard-ut_index PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_include_directories(ydb-core-tx-schemeshard-ut_index PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard ) -target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC +target_link_libraries(ydb-core-tx-schemeshard-ut_index PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -23,7 +23,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC core-testlib-default tx-schemeshard-ut_helpers ) -target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_link_options(ydb-core-tx-schemeshard-ut_index PRIVATE -ldl -lrt -Wl,--no-as-needed @@ -33,21 +33,22 @@ target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE -lrt -ldl ) -target_sources(ydb-core-tx-schemeshard-ut_async_index PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_async_index/ut_async_index.cpp +target_sources(ydb-core-tx-schemeshard-ut_index PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp ) set_property( TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY SPLIT_FACTOR 60 ) add_yunittest( NAME - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_ARG --print-before-suite --print-before-test @@ -57,26 +58,26 @@ add_yunittest( ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY LABELS MEDIUM ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY PROCESSORS 1 ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY TIMEOUT 600 ) -target_allocator(ydb-core-tx-schemeshard-ut_async_index +target_allocator(ydb-core-tx-schemeshard-ut_index cpp-malloc-jemalloc ) -vcs_info(ydb-core-tx-schemeshard-ut_async_index) +vcs_info(ydb-core-tx-schemeshard-ut_index) diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux-x86_64.txt b/ydb/core/tx/schemeshard/ut_index/CMakeLists.linux-x86_64.txt index 552428cf1c..b419d0a219 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.linux-x86_64.txt +++ b/ydb/core/tx/schemeshard/ut_index/CMakeLists.linux-x86_64.txt @@ -7,14 +7,14 @@ -add_executable(ydb-core-tx-schemeshard-ut_async_index) -target_compile_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +add_executable(ydb-core-tx-schemeshard-ut_index) +target_compile_options(ydb-core-tx-schemeshard-ut_index PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_include_directories(ydb-core-tx-schemeshard-ut_index PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard ) -target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC +target_link_libraries(ydb-core-tx-schemeshard-ut_index PUBLIC contrib-libs-linux-headers contrib-libs-cxxsupp yutil @@ -24,7 +24,7 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC core-testlib-default tx-schemeshard-ut_helpers ) -target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_link_options(ydb-core-tx-schemeshard-ut_index PRIVATE -ldl -lrt -Wl,--no-as-needed @@ -34,21 +34,22 @@ target_link_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE -lrt -ldl ) -target_sources(ydb-core-tx-schemeshard-ut_async_index PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_async_index/ut_async_index.cpp +target_sources(ydb-core-tx-schemeshard-ut_index PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp ) set_property( TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY SPLIT_FACTOR 60 ) add_yunittest( NAME - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_ARG --print-before-suite --print-before-test @@ -58,27 +59,27 @@ add_yunittest( ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY LABELS MEDIUM ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY PROCESSORS 1 ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY TIMEOUT 600 ) -target_allocator(ydb-core-tx-schemeshard-ut_async_index +target_allocator(ydb-core-tx-schemeshard-ut_index cpp-malloc-tcmalloc libs-tcmalloc-no_percpu_cache ) -vcs_info(ydb-core-tx-schemeshard-ut_async_index) +vcs_info(ydb-core-tx-schemeshard-ut_index) diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.txt b/ydb/core/tx/schemeshard/ut_index/CMakeLists.txt index f8b31df0c1..f8b31df0c1 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.txt +++ b/ydb/core/tx/schemeshard/ut_index/CMakeLists.txt diff --git a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.windows-x86_64.txt b/ydb/core/tx/schemeshard/ut_index/CMakeLists.windows-x86_64.txt index 719ae06c67..715928efb8 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/CMakeLists.windows-x86_64.txt +++ b/ydb/core/tx/schemeshard/ut_index/CMakeLists.windows-x86_64.txt @@ -7,14 +7,14 @@ -add_executable(ydb-core-tx-schemeshard-ut_async_index) -target_compile_options(ydb-core-tx-schemeshard-ut_async_index PRIVATE +add_executable(ydb-core-tx-schemeshard-ut_index) +target_compile_options(ydb-core-tx-schemeshard-ut_index PRIVATE -DUSE_CURRENT_UDF_ABI_VERSION ) -target_include_directories(ydb-core-tx-schemeshard-ut_async_index PRIVATE +target_include_directories(ydb-core-tx-schemeshard-ut_index PRIVATE ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard ) -target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC +target_link_libraries(ydb-core-tx-schemeshard-ut_index PUBLIC contrib-libs-cxxsupp yutil library-cpp-cpuid_check @@ -23,21 +23,22 @@ target_link_libraries(ydb-core-tx-schemeshard-ut_async_index PUBLIC core-testlib-default tx-schemeshard-ut_helpers ) -target_sources(ydb-core-tx-schemeshard-ut_async_index PRIVATE - ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_async_index/ut_async_index.cpp +target_sources(ydb-core-tx-schemeshard-ut_index PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp ) set_property( TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY SPLIT_FACTOR 60 ) add_yunittest( NAME - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_TARGET - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index TEST_ARG --print-before-suite --print-before-test @@ -47,26 +48,26 @@ add_yunittest( ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY LABELS MEDIUM ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY PROCESSORS 1 ) set_yunittest_property( TEST - ydb-core-tx-schemeshard-ut_async_index + ydb-core-tx-schemeshard-ut_index PROPERTY TIMEOUT 600 ) -target_allocator(ydb-core-tx-schemeshard-ut_async_index +target_allocator(ydb-core-tx-schemeshard-ut_index system_allocator ) -vcs_info(ydb-core-tx-schemeshard-ut_async_index) +vcs_info(ydb-core-tx-schemeshard-ut_index) diff --git a/ydb/core/tx/schemeshard/ut_async_index/ut_async_index.cpp b/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp index 97ae68c13c..7c22b595ca 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/ut_async_index.cpp +++ b/ydb/core/tx/schemeshard/ut_index/ut_async_index.cpp @@ -25,6 +25,12 @@ Y_UNIT_TEST_SUITE(TAsyncIndexTests) { } )"); env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndex"), + {NLs::PathExist, + NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobalAsync), + NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), + NLs::IndexKeys({"indexed"})}); } Y_UNIT_TEST(OnlineBuild) { diff --git a/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp b/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp new file mode 100644 index 0000000000..fe0dbae9ae --- /dev/null +++ b/ydb/core/tx/schemeshard/ut_index/ut_unique_index.cpp @@ -0,0 +1,35 @@ +#include <ydb/core/tx/schemeshard/ut_helpers/helpers.h> +#include <ydb/core/testlib/tablet_helpers.h> + +using namespace NKikimr; +using namespace NSchemeShard; +using namespace NSchemeShardUT_Private; + +Y_UNIT_TEST_SUITE(TUniqueIndexTests) { + Y_UNIT_TEST(CreateTable) { + TTestBasicRuntime runtime; + TTestEnv env(runtime); + ui64 txId = 100; + + TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"( + TableDescription { + Name: "Table" + Columns { Name: "key" Type: "Uint64" } + Columns { Name: "indexed" Type: "Uint64" } + KeyColumnNames: ["key"] + } + IndexDescription { + Name: "UserDefinedIndex" + KeyColumnNames: ["indexed"] + Type: EIndexTypeGlobalUnique + } + )"); + env.TestWaitNotification(runtime, txId); + + TestDescribeResult(DescribePrivatePath(runtime, "/MyRoot/Table/UserDefinedIndex"), + {NLs::PathExist, + NLs::IndexType(NKikimrSchemeOp::EIndexTypeGlobalUnique), + NLs::IndexState(NKikimrSchemeOp::EIndexStateReady), + NLs::IndexKeys({"indexed"})}); + } +} diff --git a/ydb/core/tx/schemeshard/ut_async_index/ya.make b/ydb/core/tx/schemeshard/ut_index/ya.make index c257972575..0dfb5a1a98 100644 --- a/ydb/core/tx/schemeshard/ut_async_index/ya.make +++ b/ydb/core/tx/schemeshard/ut_index/ya.make @@ -20,6 +20,7 @@ PEERDIR( SRCS( ut_async_index.cpp + ut_unique_index.cpp ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 1d458745bd..f4af193a0f 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -1,5 +1,4 @@ RECURSE_FOR_TESTS( - ut_async_index ut_backup ut_base ut_base_reboots @@ -17,6 +16,7 @@ RECURSE_FOR_TESTS( ut_extsubdomain ut_extsubdomain_reboots ut_filestore_reboots + ut_index ut_index_build ut_index_build_reboots ut_login diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp index 4aa1892f90..ee988f33f5 100644 --- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp +++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp @@ -190,7 +190,8 @@ public: IComputationExternalNode* GetEntryPoint(size_t index, bool require) { MKQL_ENSURE(index < Runtime2ComputationEntryPoints.size() && (!require || Runtime2ComputationEntryPoints[index]), - "Pattern nodes can not get computation node by index: " << index << ", require: " << require); + "Pattern nodes can not get computation node by index: " << index << ", require: " << require + << ", Runtime2ComputationEntryPoints size: " << Runtime2ComputationEntryPoints.size()); return Runtime2ComputationEntryPoints[index]; } |