diff options
author | aidarsamer <aidarsamer@ydb.tech> | 2022-11-10 11:27:06 +0300 |
---|---|---|
committer | aidarsamer <aidarsamer@ydb.tech> | 2022-11-10 11:27:06 +0300 |
commit | 56418f7c4453670d9fae2c1329fe38c5d81d4563 (patch) | |
tree | 9f61213141b0de9e55bcd78ffdfa6d8b0a4ebd25 | |
parent | 5bb7bca00ab0df5e03bf689cacea93090b9d9c45 (diff) | |
download | ydb-56418f7c4453670d9fae2c1329fe38c5d81d4563.tar.gz |
Refactor KqpOlapFilter to exclude Input from conditions. Add OLAP logic operations.
Refactor KqpOlapFilters to remove Input field.
Add OLAP logic operations
-rw-r--r-- | ydb/core/kqp/compile/kqp_olap_compiler.cpp | 18 | ||||
-rw-r--r-- | ydb/core/kqp/expr_nodes/kqp_expr_nodes.json | 49 | ||||
-rw-r--r-- | ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp | 128 | ||||
-rw-r--r-- | ydb/core/kqp/prepare/kqp_type_ann.cpp | 181 | ||||
-rw-r--r-- | ydb/core/kqp/ut/kqp_olap_ut.cpp | 2 |
5 files changed, 183 insertions, 195 deletions
diff --git a/ydb/core/kqp/compile/kqp_olap_compiler.cpp b/ydb/core/kqp/compile/kqp_olap_compiler.cpp index e0d180dff6..b552594203 100644 --- a/ydb/core/kqp/compile/kqp_olap_compiler.cpp +++ b/ydb/core/kqp/compile/kqp_olap_compiler.cpp @@ -256,15 +256,15 @@ TProgram::TAssignment* CompileComparison(const TKqpOlapFilterCompare& comparison ui32 function = TProgram::TAssignment::FUNC_UNSPECIFIED; - if (comparison.Maybe<TKqpOlapFilterEqual>()) { + if (comparison.Operator() == "eq") { function = TProgram::TAssignment::FUNC_CMP_EQUAL; - } else if (comparison.Maybe<TKqpOlapFilterLess>()) { + } else if (comparison.Operator() == "lt") { function = TProgram::TAssignment::FUNC_CMP_LESS; - } else if (comparison.Maybe<TKqpOlapFilterLessOrEqual>()) { + } else if (comparison.Operator() == "lte") { function = TProgram::TAssignment::FUNC_CMP_LESS_EQUAL; - } else if (comparison.Maybe<TKqpOlapFilterGreater>()) { + } else if (comparison.Operator() == "gt") { function = TProgram::TAssignment::FUNC_CMP_GREATER; - } else if (comparison.Maybe<TKqpOlapFilterGreaterOrEqual>()) { + } else if (comparison.Operator() == "gte") { function = TProgram::TAssignment::FUNC_CMP_GREATER_EQUAL; } @@ -345,7 +345,7 @@ TProgram::TAssignment* CompileCondition(const TExprBase& condition, TKqpOlapComp return CompileExists(maybeExists.Cast(), ctx); } - if (auto maybeNot = condition.Maybe<TCoNot>()) { + if (auto maybeNot = condition.Maybe<TKqpOlapNot>()) { // Not is a special way in case it has only one child TProgram::TAssignment *value = CompileCondition(maybeNot.Cast().Value(), ctx); @@ -360,11 +360,11 @@ TProgram::TAssignment* CompileCondition(const TExprBase& condition, TKqpOlapComp ui32 function = TProgram::TAssignment::FUNC_UNSPECIFIED; - if (condition.Maybe<TCoAnd>()) { + if (condition.Maybe<TKqpOlapAnd>()) { function = TProgram::TAssignment::FUNC_BINARY_AND; - } else if (condition.Maybe<TCoOr>()) { + } else if (condition.Maybe<TKqpOlapOr>()) { function = TProgram::TAssignment::FUNC_BINARY_OR; - } else if (condition.Maybe<TCoXor>()) { + } else if (condition.Maybe<TKqpOlapXor>()) { function = TProgram::TAssignment::FUNC_BINARY_XOR; } else { YQL_ENSURE(false, "Unsuppoted logical operation: " << condition.Ptr()->Content()); diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json index 9b5d293456..051a73d2f5 100644 --- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json +++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json @@ -400,45 +400,44 @@ ] }, { - "Name": "TKqpOlapFilterCompare", - "Base": "TKqpOlapOperationBase", - "Match": {"Type": "CallableBase"}, + "Name": "TKqpOlapNot", + "Base": "TCallable", + "Match": {"Type": "Callable", "Name": "KqpOlapNot"}, "Children": [ - {"Index": 1, "Name": "Left", "Type": "TExprBase"}, - {"Index": 2, "Name": "Right", "Type": "TExprBase"} + {"Index": 0, "Name": "Value", "Type": "TExprBase"} ] }, { - "Name": "TKqpOlapFilterEqual", - "Base": "TKqpOlapFilterCompare", - "Match": {"Type": "Callable", "Name": "KqpOlapFilterEqual"} - }, - { - "Name": "TKqpOlapFilterLess", - "Base": "TKqpOlapFilterCompare", - "Match": {"Type": "Callable", "Name": "KqpOlapFilterLess"} + "Name": "TKqpOlapAnd", + "VarArgBase": "TExprBase", + "Match": {"Type": "Callable", "Name": "KqpOlapAnd"} }, { - "Name": "TKqpOlapFilterLessOrEqual", - "Base": "TKqpOlapFilterCompare", - "Match": {"Type": "Callable", "Name": "KqpOlapFilterLessOrEqual"} + "Name": "TKqpOlapOr", + "VarArgBase": "TExprBase", + "Match": {"Type": "Callable", "Name": "KqpOlapOr"} }, { - "Name": "TKqpOlapFilterGreater", - "Base": "TKqpOlapFilterCompare", - "Match": {"Type": "Callable", "Name": "KqpOlapFilterGreater"} + "Name": "TKqpOlapXor", + "VarArgBase": "TExprBase", + "Match": {"Type": "Callable", "Name": "KqpOlapXor"} }, { - "Name": "TKqpOlapFilterGreaterOrEqual", - "Base": "TKqpOlapFilterCompare", - "Match": {"Type": "Callable", "Name": "KqpOlapFilterGreaterOrEqual"} + "Name": "TKqpOlapFilterCompare", + "Base": "TExprBase", + "Match": {"Type": "Tuple", "Name": "KqpOlapFilterCompare"}, + "Children": [ + {"Index": 0, "Name": "Operator", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Left", "Type": "TExprBase"}, + {"Index": 2, "Name": "Right", "Type": "TExprBase"} + ] }, { "Name": "TKqpOlapFilterExists", - "Base": "TKqpOlapOperationBase", - "Match": {"Type": "Callable", "Name": "TKqpOlapFilterExists"}, + "Base": "TExprBase", + "Match": {"Type": "Tuple", "Name": "KqpOlapFilterExists"}, "Children": [ - {"Index": 1, "Name": "Column", "Type": "TExprBase"} + {"Index": 0, "Name": "Column", "Type": "TCoAtom"} ] }, { diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp index 432483b606..fd514740c8 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp @@ -12,6 +12,10 @@ namespace { static TMaybeNode<TExprBase> NullNode = TMaybeNode<TExprBase>(); +bool IsFalseLiteral(TExprBase node) { + return node.Maybe<TCoBool>() && !FromString<bool>(node.Cast<TCoBool>().Literal().Value()); +} + bool IsSupportedPredicate(const TCoCompare& predicate) { if (predicate.Maybe<TCoCmpEqual>()) { return true; @@ -368,7 +372,7 @@ TVector<std::pair<TExprBase, TExprBase>> ExtractComparisonParameters(const TCoCo } TExprBase BuildOneElementComparison(const std::pair<TExprBase, TExprBase>& parameter, const TCoCompare& predicate, - TExprContext& ctx, TPositionHandle pos, const TExprBase& input, bool forceStrictComparison) + TExprContext& ctx, TPositionHandle pos, bool forceStrictComparison) { auto isNull = [](const TExprBase& node) { if (node.Maybe<TCoNull>()) { @@ -391,41 +395,23 @@ TExprBase BuildOneElementComparison(const std::pair<TExprBase, TExprBase>& param .Done(); } - if (predicate.Maybe<TCoCmpEqual>()) { - return Build<TKqpOlapFilterEqual>(ctx, pos) - .Input(input) - .Left(parameter.first) - .Right(parameter.second) - .Done(); - } - - if (predicate.Maybe<TCoCmpLess>() || (predicate.Maybe<TCoCmpLessOrEqual>() && forceStrictComparison)) { - return Build<TKqpOlapFilterLess>(ctx, pos) - .Input(input) - .Left(parameter.first) - .Right(parameter.second) - .Done(); - } - - if (predicate.Maybe<TCoCmpLessOrEqual>() && !forceStrictComparison) { - return Build<TKqpOlapFilterLessOrEqual>(ctx, pos) - .Input(input) - .Left(parameter.first) - .Right(parameter.second) - .Done(); - } - - if (predicate.Maybe<TCoCmpGreater>() || (predicate.Maybe<TCoCmpGreaterOrEqual>() && forceStrictComparison)) { - return Build<TKqpOlapFilterGreater>(ctx, pos) - .Input(input) - .Left(parameter.first) - .Right(parameter.second) - .Done(); - } + std::string compareOperator = ""; - if (predicate.Maybe<TCoCmpGreaterOrEqual>() && !forceStrictComparison) { - return Build<TKqpOlapFilterGreaterOrEqual>(ctx, pos) - .Input(input) + if (predicate.Maybe<TCoCmpEqual>()) { + compareOperator = "eq"; + } else if (predicate.Maybe<TCoCmpLess>() || (predicate.Maybe<TCoCmpLessOrEqual>() && forceStrictComparison)) { + compareOperator = "lt"; + } else if (predicate.Maybe<TCoCmpLessOrEqual>() && !forceStrictComparison) { + compareOperator = "lte"; + } else if (predicate.Maybe<TCoCmpGreater>() || (predicate.Maybe<TCoCmpGreaterOrEqual>() && forceStrictComparison)) { + compareOperator = "gt"; + } if (predicate.Maybe<TCoCmpGreaterOrEqual>() && !forceStrictComparison) { + compareOperator = "gte"; + } + + if (!compareOperator.empty()) { + return Build<TKqpOlapFilterCompare>(ctx, pos) + .Operator(ctx.NewAtom(pos, compareOperator)) .Left(parameter.first) .Right(parameter.second) .Done(); @@ -433,39 +419,49 @@ TExprBase BuildOneElementComparison(const std::pair<TExprBase, TExprBase>& param YQL_ENSURE(predicate.Maybe<TCoCmpNotEqual>(), "Unsupported comparison node: " << predicate.Ptr()->Content()); - return Build<TCoNot>(ctx, pos) - .Value<TKqpOlapFilterEqual>() - .Input(input) + return Build<TKqpOlapNot>(ctx, pos) + .Value<TKqpOlapFilterCompare>() + .Operator(ctx.NewAtom(pos, "eq")) .Left(parameter.first) .Right(parameter.second) .Build() .Done(); } -TExprBase ComparisonPushdown(const TVector<std::pair<TExprBase, TExprBase>>& parameters, const TCoCompare& predicate, - TExprContext& ctx, TPositionHandle pos, const TExprBase& input) +TMaybeNode<TExprBase> ComparisonPushdown(const TVector<std::pair<TExprBase, TExprBase>>& parameters, const TCoCompare& predicate, + TExprContext& ctx, TPositionHandle pos) { ui32 conditionsCount = parameters.size(); if (conditionsCount == 1) { - return BuildOneElementComparison(parameters[0], predicate, ctx, pos, input, false); + auto condition = BuildOneElementComparison(parameters[0], predicate, ctx, pos, false); + return IsFalseLiteral(condition) ? NullNode : condition; } if (predicate.Maybe<TCoCmpEqual>() || predicate.Maybe<TCoCmpNotEqual>()) { TVector<TExprBase> conditions; conditions.reserve(conditionsCount); + bool hasFalseCondition = false; for (ui32 i = 0; i < conditionsCount; ++i) { - conditions.emplace_back(BuildOneElementComparison(parameters[i], predicate, ctx, pos, input, false)); + auto condition = BuildOneElementComparison(parameters[i], predicate, ctx, pos, false); + if (IsFalseLiteral(condition)) { + hasFalseCondition = true; + } else { + conditions.emplace_back(condition); + } } if (predicate.Maybe<TCoCmpEqual>()) { - return Build<TCoAnd>(ctx, pos) + if (hasFalseCondition) { + return NullNode; + } + return Build<TKqpOlapAnd>(ctx, pos) .Add(conditions) .Done(); } - return Build<TCoOr>(ctx, pos) + return Build<TKqpOlapOr>(ctx, pos) .Add(conditions) .Done(); } @@ -473,29 +469,35 @@ TExprBase ComparisonPushdown(const TVector<std::pair<TExprBase, TExprBase>>& par TVector<TExprBase> orConditions; orConditions.reserve(conditionsCount); - // Here we can be only whe comparing tuples lexicographically + // Here we can be only when comparing tuples lexicographically for (ui32 i = 0; i < conditionsCount; ++i) { TVector<TExprBase> andConditions; andConditions.reserve(conditionsCount); // We need strict < and > in beginning columns except the last one // For example: (c1, c2, c3) >= (1, 2, 3) ==> (c1 > 1) OR (c2 > 2 AND c1 = 1) OR (c3 >= 3 AND c2 = 2 AND c1 = 1) - andConditions.emplace_back(BuildOneElementComparison(parameters[i], predicate, ctx, pos, input, i < conditionsCount - 1)); + auto condition = BuildOneElementComparison(parameters[i], predicate, ctx, pos, i < conditionsCount - 1); + if (IsFalseLiteral(condition)) { + continue; + } + andConditions.emplace_back(condition); for (ui32 j = 0; j < i; ++j) { - andConditions.emplace_back(Build<TKqpOlapFilterEqual>(ctx, pos) - .Input(input) + andConditions.emplace_back(Build<TKqpOlapFilterCompare>(ctx, pos) + .Operator(ctx.NewAtom(pos, "eq")) .Left(parameters[j].first) .Right(parameters[j].second) .Done()); } - orConditions.emplace_back(Build<TCoAnd>(ctx, pos) - .Add(std::move(andConditions)) - .Done()); + orConditions.emplace_back( + Build<TKqpOlapAnd>(ctx, pos) + .Add(std::move(andConditions)) + .Done() + ); } - return Build<TCoOr>(ctx, pos) + return Build<TKqpOlapOr>(ctx, pos) .Add(std::move(orConditions)) .Done(); } @@ -528,8 +530,7 @@ TMaybeNode<TCoAtomList> BuildColumnsFromLambda(const TCoLambda& lambda, TExprCon } #endif -TMaybeNode<TExprBase> ExistsPushdown(const TCoExists& exists, TExprContext& ctx, TPositionHandle pos, - const TExprNode* lambdaArg, const TExprBase& input) +TMaybeNode<TExprBase> ExistsPushdown(const TCoExists& exists, TExprContext& ctx, TPositionHandle pos, const TExprNode* lambdaArg) { auto maybeMember = exists.Optional().Maybe<TCoMember>(); @@ -544,13 +545,12 @@ TMaybeNode<TExprBase> ExistsPushdown(const TCoExists& exists, TExprContext& ctx, auto columnName = maybeMember.Cast().Name(); return Build<TKqpOlapFilterExists>(ctx, pos) - .Input(input) .Column(columnName) .Done(); } TMaybeNode<TExprBase> SafeCastPredicatePushdown(const TCoFlatMap& flatmap, - TExprContext& ctx, TPositionHandle pos, const TExprNode* lambdaArg, const TExprBase& input) + TExprContext& ctx, TPositionHandle pos, const TExprNode* lambdaArg) { /* * There are three ways of comparison in following format: @@ -610,7 +610,7 @@ TMaybeNode<TExprBase> SafeCastPredicatePushdown(const TCoFlatMap& flatmap, out.emplace_back(std::move(std::make_pair(left[i], right[i]))); } - return ComparisonPushdown(parameters, predicate, ctx, pos, input); + return ComparisonPushdown(parameters, predicate, ctx, pos); } TMaybeNode<TExprBase> SimplePredicatePushdown(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos, @@ -626,7 +626,7 @@ TMaybeNode<TExprBase> SimplePredicatePushdown(const TCoCompare& predicate, TExpr return NullNode; } - return ComparisonPushdown(parameters, predicate, ctx, pos, input); + return ComparisonPushdown(parameters, predicate, ctx, pos); } @@ -644,7 +644,7 @@ TMaybeNode<TExprBase> CoalescePushdown(const TCoCoalesce& coalesce, TExprContext auto maybeFlatmap = coalesce.Predicate().Maybe<TCoFlatMap>(); if (maybeFlatmap.IsValid()) { - return SafeCastPredicatePushdown(maybeFlatmap.Cast(), ctx, pos, lambdaArg, input); + return SafeCastPredicatePushdown(maybeFlatmap.Cast(), ctx, pos, lambdaArg); } auto maybePredicate = coalesce.Predicate().Maybe<TCoCompare>(); @@ -668,7 +668,7 @@ TMaybeNode<TExprBase> PredicatePushdown(const TExprBase& predicate, TExprContext auto maybeExists = predicate.Maybe<TCoExists>(); if (maybeExists.IsValid()) { - return ExistsPushdown(maybeExists.Cast(), ctx, pos, lambdaArg, input); + return ExistsPushdown(maybeExists.Cast(), ctx, pos, lambdaArg); } if (predicate.Maybe<TCoNot>()) { @@ -679,7 +679,7 @@ TMaybeNode<TExprBase> PredicatePushdown(const TExprBase& predicate, TExprContext return NullNode; } - return Build<TCoNot>(ctx, pos) + return Build<TKqpOlapNot>(ctx, pos) .Value(pushedNot.Cast()) .Done(); } @@ -702,20 +702,20 @@ TMaybeNode<TExprBase> PredicatePushdown(const TExprBase& predicate, TExprContext } if (predicate.Maybe<TCoAnd>()) { - return Build<TCoAnd>(ctx, pos) + return Build<TKqpOlapAnd>(ctx, pos) .Add(pushedOps) .Done(); } if (predicate.Maybe<TCoOr>()) { - return Build<TCoOr>(ctx, pos) + return Build<TKqpOlapOr>(ctx, pos) .Add(pushedOps) .Done(); } Y_VERIFY_DEBUG(predicate.Maybe<TCoXor>()); - return Build<TCoXor>(ctx, pos) + return Build<TKqpOlapXor>(ctx, pos) .Add(pushedOps) .Done(); } diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp index d419fbd1a8..abc11f177e 100644 --- a/ydb/core/kqp/prepare/kqp_type_ann.cpp +++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp @@ -698,118 +698,118 @@ TStatus AnnotateDeleteRows(const TExprNode::TPtr& node, TExprContext& ctx, const return TStatus::Ok; } -TStatus AnnotateOlapFilter(const TExprNode::TPtr& node, TExprContext& ctx) { - if (!EnsureArgsCount(*node, 2, ctx)) { - return TStatus::Error; - } - - auto* input = node->Child(TKqpOlapFilter::idx_Input); - - const TTypeAnnotationNode* itemType; - if (!EnsureNewSeqType<false, false, true>(*input, ctx, &itemType)) { - return TStatus::Error; - } - - if (!EnsureStructType(input->Pos(), *itemType, ctx)) { - return TStatus::Error; - } - - if (!EnsureSpecificDataType(*node->Child(TKqpOlapFilter::idx_Condition), EDataSlot::Bool, ctx)) { +TStatus AnnotateOlapUnaryLogicOperator(const TExprNode::TPtr& node, TExprContext& ctx) { + if (!EnsureArgsCount(*node, 1, ctx)) { return TStatus::Error; } - node->SetTypeAnn(input->GetTypeAnn()); + node->SetTypeAnn(ctx.MakeType<TUnitExprType>()); return TStatus::Ok; } -TStatus AnnotateOlapFilterCompare(const TExprNode::TPtr& node, TExprContext& ctx) { - if (!EnsureArgsCount(*node, 3, ctx)) { +TStatus AnnotateOlapBinaryLogicOperator(const TExprNode::TPtr& node, TExprContext& ctx) { + if (!EnsureMinArgsCount(*node, 1, ctx)) { return TStatus::Error; } - auto* input = node->Child(TKqpOlapFilterCompare::idx_Input); - - const TTypeAnnotationNode* itemType; - - if (!EnsureNewSeqType<false, false, true>(*input, ctx, &itemType)) { - return TStatus::Error; - } - - if (!EnsureStructType(input->Pos(), *itemType, ctx)) { - return TStatus::Error; - } + node->SetTypeAnn(ctx.MakeType<TUnitExprType>()); + return TStatus::Ok; +} - auto validateNode = [itemType, &ctx](TExprNode* node) { - // Column name, validate that it is present in Input node - if (TCoAtom::Match(node)) { - auto rowType = itemType->Cast<TStructExprType>(); +bool ValidateOlapFilterConditions(const TExprNode* node, const TStructExprType* itemType, TExprContext& ctx) { + YQL_ENSURE(itemType); - if (rowType->FindItem(node->Content())) { - return true; + if (TKqpOlapAnd::Match(node) || TKqpOlapOr::Match(node) || TKqpOlapXor::Match(node) || TKqpOlapNot::Match(node)) { + bool res = true; + for (auto arg : node->ChildrenList()) { + res &= ValidateOlapFilterConditions(arg.Get(), itemType, ctx); + if (!res) { + break; } - - ctx.AddError(TIssue( - ctx.GetPosition(node->Pos()), - TStringBuilder() << "Missing column in input type: " << node->Content() + } + return res; + } else if (TKqpOlapFilterCompare::Match(node)) { + if (!EnsureArgsCount(*node, 3, ctx)) { + return false; + } + auto op = node->Child(TKqpOlapFilterCompare::idx_Operator); + if (!EnsureAtom(*op, ctx)) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Expected string as operator in OLAP comparison filter, got: " << op->Content() )); - return false; } - - // Null argument for IS NULL/NOT NULL - if (TCoNull::Match(node)) { - return true; + auto opStr = op->Content(); + if (opStr != "eq"sv && opStr != "neq"sv && opStr != "lt"sv && opStr != "lte"sv && opStr != "gt"sv && opStr != "gte"sv) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Expected one of eq/neq/lt/lte/gt/gte operators in OLAP comparison filter, got: " << op->Content() + )); + return false; } - - // Incoming parameter - if (TCoParameter::Match(node)) { - return true; + return ValidateOlapFilterConditions(node->Child(TKqpOlapFilterCompare::idx_Left), itemType, ctx) + && ValidateOlapFilterConditions(node->Child(TKqpOlapFilterCompare::idx_Right), itemType, ctx); + } else if (TKqpOlapFilterExists::Match(node)) { + if (!EnsureArgsCount(*node, 1, ctx)) { + return false; } - - // Any supported literal - if (TCoDataCtor::Match(node)) { - return true; + auto column = node->Child(TKqpOlapFilterExists::idx_Column); + if (!EnsureAtom(*column, ctx)) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Expected column in OLAP Exists filter, got: " << column->Content() + )); + return false; } + return ValidateOlapFilterConditions(column, itemType, ctx); + } - // SafeCast, the checks about validity should be placed in kqp_opt_phy_olap_filter.cpp - if (TCoSafeCast::Match(node)) { + // Column name, validate that it is present in Input node + if (TCoAtom::Match(node)) { + if (itemType->FindItem(node->Content())) { return true; } - ctx.AddError(TIssue( - ctx.GetPosition(node->Pos()), - TStringBuilder() - << "Expected literal or column as OLAP filter value, got: " << node->Content() + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Missing column in input type: " << node->Content() )); return false; - }; + } - auto leftNode = node->Child(TKqpOlapFilterCompare::idx_Left); - auto rightNode = node->Child(TKqpOlapFilterCompare::idx_Right); + // Null argument for IS NULL/NOT NULL + if (TCoNull::Match(node)) { + return true; + } - if (!validateNode(leftNode)) { - return TStatus::Error; + // Incoming parameter + if (TCoParameter::Match(node)) { + return true; } - if (!validateNode(rightNode)) { - return TStatus::Error; + // Any supported literal + if (TCoDataCtor::Match(node)) { + return true; + } + + // SafeCast, the checks about validity should be placed in kqp_opt_phy_olap_filter.cpp + if (TCoSafeCast::Match(node)) { + return true; } - node->SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::Bool)); + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Expected literal or column as OLAP filter value, got: " << node->Content() + )); - return TStatus::Ok; + return false; } -TStatus AnnotateOlapFilterExists(const TExprNode::TPtr& node, TExprContext& ctx) { +TStatus AnnotateOlapFilter(const TExprNode::TPtr& node, TExprContext& ctx) { if (!EnsureArgsCount(*node, 2, ctx)) { return TStatus::Error; } - auto* input = node->Child(TKqpOlapFilterExists::idx_Input); + auto* input = node->Child(TKqpOlapFilter::idx_Input); const TTypeAnnotationNode* itemType; - if (!EnsureNewSeqType<false, false, true>(*input, ctx, &itemType)) { return TStatus::Error; } @@ -818,28 +818,11 @@ TStatus AnnotateOlapFilterExists(const TExprNode::TPtr& node, TExprContext& ctx) return TStatus::Error; } - auto column = node->Child(TKqpOlapFilterExists::idx_Column); - - if (!EnsureAtom(*column, ctx)) { - ctx.AddError(TIssue( - ctx.GetPosition(node->Pos()), - TStringBuilder() - << "Expected column in OLAP Exists filter, got: " << column->Content() - )); - + if (!ValidateOlapFilterConditions(node->Child(TKqpOlapFilter::idx_Condition), itemType->Cast<TStructExprType>(), ctx)) { return TStatus::Error; } - auto rowType = itemType->Cast<TStructExprType>(); - - if (!rowType->FindItem(column->Content())) { - ctx.AddError(TIssue( - ctx.GetPosition(node->Pos()), - TStringBuilder() << "Missing column in OLAP Exists filter in input type: " << column->Content() - )); - } - - node->SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::Bool)); + node->SetTypeAnn(input->GetTypeAnn()); return TStatus::Ok; } @@ -1266,16 +1249,20 @@ TAutoPtr<IGraphTransformer> CreateKqpTypeAnnotationTransformer(const TString& cl return AnnotateDeleteRows(input, ctx, cluster, *tablesData); } - if (TKqpOlapFilter::Match(input.Get())) { - return AnnotateOlapFilter(input, ctx); + if (TKqpOlapAnd::Match(input.Get()) + || TKqpOlapOr::Match(input.Get()) + || TKqpOlapXor::Match(input.Get()) + ) + { + return AnnotateOlapBinaryLogicOperator(input, ctx); } - if (TKqpOlapFilterCompare::Match(input.Get())) { - return AnnotateOlapFilterCompare(input, ctx); + if (TKqpOlapNot::Match(input.Get())) { + return AnnotateOlapUnaryLogicOperator(input, ctx); } - if (TKqpOlapFilterExists::Match(input.Get())) { - return AnnotateOlapFilterExists(input, ctx); + if (TKqpOlapFilter::Match(input.Get())) { + return AnnotateOlapFilter(input, ctx); } if (TKqpOlapAgg::Match(input.Get())) { diff --git a/ydb/core/kqp/ut/kqp_olap_ut.cpp b/ydb/core/kqp/ut/kqp_olap_ut.cpp index 40373be49b..91063cd9f8 100644 --- a/ydb/core/kqp/ut/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/kqp_olap_ut.cpp @@ -1104,6 +1104,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) { R"(`level` > NULL)", R"(`timestamp` >= CAST(3000001 AS Timestamp))", R"(`level` >= CAST("2" As Uint32))", + R"(`level` = NULL)", + R"(`level` > NULL)", }; auto buildQuery = [](const TString& predicate, bool pushEnabled) { |