diff options
author | vokayndzop <[email protected]> | 2025-01-29 16:25:37 +0300 |
---|---|---|
committer | vokayndzop <[email protected]> | 2025-01-29 17:18:25 +0300 |
commit | be07767ad39d693f1f8165c85adf05f925ff84bf (patch) | |
tree | 827003dadb10987e5aba89309eddb7902034ca31 /yql/essentials/sql/v1 | |
parent | 6a31c2b82c0136d2a116b7be4e155d1898d4e7eb (diff) |
MR: aggregation in MEASURES
commit_hash:5357736eed7a221ff5844d4351abe23e65930632
Diffstat (limited to 'yql/essentials/sql/v1')
-rw-r--r-- | yql/essentials/sql/v1/builtin.cpp | 34 | ||||
-rw-r--r-- | yql/essentials/sql/v1/context.h | 43 | ||||
-rw-r--r-- | yql/essentials/sql/v1/match_recognize.cpp | 530 | ||||
-rw-r--r-- | yql/essentials/sql/v1/match_recognize.h | 145 | ||||
-rw-r--r-- | yql/essentials/sql/v1/node.cpp | 12 | ||||
-rw-r--r-- | yql/essentials/sql/v1/node.h | 3 | ||||
-rw-r--r-- | yql/essentials/sql/v1/source.cpp | 2 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_expression.cpp | 45 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_expression.h | 2 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_match_recognize.cpp | 662 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_match_recognize.h | 36 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_match_recognize_ut.cpp | 24 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_ut.cpp | 39 | ||||
-rw-r--r-- | yql/essentials/sql/v1/sql_ut_antlr4.cpp | 39 |
14 files changed, 958 insertions, 658 deletions
diff --git a/yql/essentials/sql/v1/builtin.cpp b/yql/essentials/sql/v1/builtin.cpp index c9953c6e0d0..8e49ba39f22 100644 --- a/yql/essentials/sql/v1/builtin.cpp +++ b/yql/essentials/sql/v1/builtin.cpp @@ -3149,10 +3149,6 @@ struct TBuiltinFuncData { // Hopping intervals time functions {"hopstart", BuildSimpleBuiltinFactoryCallback<THoppingTime<true>>()}, {"hopend", BuildSimpleBuiltinFactoryCallback<THoppingTime<false>>()}, - - //MatchRecognize navigation functions - {"first", BuildNamedBuiltinFactoryCallback<TMatchRecognizeNavigate>("FIRST")}, - {"last", BuildNamedBuiltinFactoryCallback<TMatchRecognizeNavigate>("LAST")}, }; return builtinFuncs; } @@ -3269,6 +3265,10 @@ struct TBuiltinFuncData { {"firstvalueignorenulls", BuildAggrFuncFactoryCallback("FirstValueIgnoreNulls", "first_value_ignore_nulls_traits_factory", {OverWindow})}, {"lastvalueignorenulls", BuildAggrFuncFactoryCallback("LastValueIgnoreNulls", "last_value_ignore_nulls_traits_factory", {OverWindow})}, {"nthvalueignorenulls", BuildAggrFuncFactoryCallback("NthValueIgnoreNulls", "nth_value_ignore_nulls_traits_factory", {OverWindow}, NTH_VALUE)}, + + // MatchRecognize navigation functions + {"first", BuildAggrFuncFactoryCallback("First", "first_traits_factory")}, + {"last", BuildAggrFuncFactoryCallback("Last", "last_traits_factory")}, }; return aggrFuncs; } @@ -3633,7 +3633,17 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec return new TInvalidBuiltin(pos, TStringBuilder() << "Unknown aggregation function: " << *args[0]->GetLiteral("String")); } - return (*aggrCallback).second(pos, args, aggMode, true).Release(); + switch (ctx.GetColumnReferenceState()) { + case EColumnRefState::MatchRecognizeMeasures: + [[fallthrough]]; + case EColumnRefState::MatchRecognizeDefine: + return new TInvalidBuiltin(pos, "Cannot use aggregation factory inside the MATCH_RECOGNIZE context"); + default: + if ("first" == aggNormalizedName || "last" == aggNormalizedName) { + return new TInvalidBuiltin(pos, "Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context"); + } + return (*aggrCallback).second(pos, args, aggMode, true); + } } if (normalizedName == "aggregateby" || normalizedName == "multiaggregateby") { @@ -3651,7 +3661,19 @@ TNodePtr BuildBuiltinFunc(TContext& ctx, TPosition pos, TString name, const TVec auto aggrCallback = aggrFuncs.find(normalizedName); if (aggrCallback != aggrFuncs.end()) { - return (*aggrCallback).second(pos, args, aggMode, false).Release(); + switch (ctx.GetColumnReferenceState()) { + case EColumnRefState::MatchRecognizeMeasures: { + auto result = (*aggrCallback).second(pos, args, aggMode, false); + return BuildMatchRecognizeVarAccess(pos, std::move(result)); + } + case EColumnRefState::MatchRecognizeDefine: + return BuildMatchRecognizeDefineAggregate(ctx.Pos(), normalizedName, args); + default: + if ("first" == normalizedName || "last" == normalizedName) { + return new TInvalidBuiltin(pos, "Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context"); + } + return (*aggrCallback).second(pos, args, aggMode, false); + } } if (aggMode == EAggregateMode::Distinct || aggMode == EAggregateMode::OverWindowDistinct) { return new TInvalidBuiltin(pos, "DISTINCT can only be used in aggregation functions"); diff --git a/yql/essentials/sql/v1/context.h b/yql/essentials/sql/v1/context.h index 85a4739eaf5..31185c0d77a 100644 --- a/yql/essentials/sql/v1/context.h +++ b/yql/essentials/sql/v1/context.h @@ -85,7 +85,9 @@ namespace NSQLTranslationV1 { Allow, AsStringLiteral, AsPgType, - MatchRecognize, + MatchRecognizeMeasures, + MatchRecognizeDefine, + MatchRecognizeDefineAggregate, }; class TContext { @@ -204,12 +206,36 @@ namespace NSQLTranslationV1 { return TopLevelColumnReferenceState; } - TStringBuf GetMatchRecognizeDefineVar() const { - YQL_ENSURE(EColumnRefState::MatchRecognize == ColumnReferenceState, - "DefineVar can only be accessed within processing of MATCH_RECOGNIZE lambdas"); + [[nodiscard]] TString GetMatchRecognizeDefineVar() const { + YQL_ENSURE(EColumnRefState::MatchRecognizeMeasures == ColumnReferenceState + || EColumnRefState::MatchRecognizeDefine == ColumnReferenceState + || EColumnRefState::MatchRecognizeDefineAggregate == ColumnReferenceState, + "MATCH_RECOGNIZE Var can only be accessed within processing of MATCH_RECOGNIZE lambdas"); return MatchRecognizeDefineVar; } + TString ExtractMatchRecognizeAggrVar() { + YQL_ENSURE(EColumnRefState::MatchRecognizeMeasures == ColumnReferenceState + || EColumnRefState::MatchRecognizeDefine == ColumnReferenceState + || EColumnRefState::MatchRecognizeDefineAggregate == ColumnReferenceState, + "MATCH_RECOGNIZE Var can only be accessed within processing of MATCH_RECOGNIZE lambdas"); + return std::exchange(MatchRecognizeAggrVar, ""); + } + + [[nodiscard]] bool SetMatchRecognizeAggrVar(TString var) { + YQL_ENSURE(EColumnRefState::MatchRecognizeMeasures == ColumnReferenceState + || EColumnRefState::MatchRecognizeDefine == ColumnReferenceState + || EColumnRefState::MatchRecognizeDefineAggregate == ColumnReferenceState, + "MATCH_RECOGNIZE Var can only be accessed within processing of MATCH_RECOGNIZE lambdas"); + if (MatchRecognizeAggrVar.empty()) { + MatchRecognizeAggrVar = std::move(var); + } else if (MatchRecognizeAggrVar != var) { + Error() << "Illegal use of aggregates or navigation operators in MATCH_RECOGNIZE"; + return false; + } + return true; + } + TVector<NSQLTranslation::TSQLHint> PullHintForToken(NYql::TPosition tokenPos); void WarnUnusedHints(); @@ -231,6 +257,7 @@ namespace NSQLTranslationV1 { EColumnRefState ColumnReferenceState = EColumnRefState::Deny; EColumnRefState TopLevelColumnReferenceState = EColumnRefState::Deny; TString MatchRecognizeDefineVar; + TString MatchRecognizeAggrVar; TString NoColumnErrorContext = "in current scope"; TVector<TBlocks*> CurrentBlocks; @@ -347,7 +374,13 @@ namespace NSQLTranslationV1 { } else { Ctx.ColumnReferenceState = state; } - YQL_ENSURE(defineVar.empty() || EColumnRefState::MatchRecognize == state, "Internal logic error"); + YQL_ENSURE( + defineVar.empty() + || EColumnRefState::MatchRecognizeMeasures == state + || EColumnRefState::MatchRecognizeDefine == state + || EColumnRefState::MatchRecognizeDefineAggregate == state, + "Internal logic error" + ); ctx.MatchRecognizeDefineVar = defineVar; } diff --git a/yql/essentials/sql/v1/match_recognize.cpp b/yql/essentials/sql/v1/match_recognize.cpp index 84a20ae2731..be2f508e879 100644 --- a/yql/essentials/sql/v1/match_recognize.cpp +++ b/yql/essentials/sql/v1/match_recognize.cpp @@ -2,111 +2,302 @@ #include "source.h" #include "context.h" +#include <util/generic/overloaded.h> + namespace NSQLTranslationV1 { namespace { -const auto VarDataName = "data"; -const auto VarMatchedVarsName = "vars"; -const auto VarLastRowIndexName = "lri"; +constexpr auto VarDataName = "data"; +constexpr auto VarMatchedVarsName = "vars"; +constexpr auto VarLastRowIndexName = "lri"; + +class TMatchRecognizeColumnAccessNode final : public TAstListNode { +public: + TMatchRecognizeColumnAccessNode(TPosition pos, TString var, TString column) + : TAstListNode(pos) + , Var(std::move(var)) + , Column(std::move(column)) { + } + + const TString* GetColumnName() const override { + return std::addressof(Column); + } + + bool DoInit(TContext& ctx, ISource* /* src */) override { + switch (ctx.GetColumnReferenceState()) { + case EColumnRefState::MatchRecognizeMeasures: + if (!ctx.SetMatchRecognizeAggrVar(Var)) { + return false; + } + Add( + "Member", + BuildAtom(Pos, "row"), + Q(Column) + ); + break; + case EColumnRefState::MatchRecognizeDefine: + if (ctx.GetMatchRecognizeDefineVar() != Var) { + ctx.Error() << "Row pattern navigation function is required"; + return false; + } + BuildLookup(VarLastRowIndexName); + break; + case EColumnRefState::MatchRecognizeDefineAggregate: + if (!ctx.SetMatchRecognizeAggrVar(Var)) { + return false; + } + BuildLookup("index"); + break; + default: + Y_ABORT("Unexpected column reference state"); + } + return true; + } + + TNodePtr DoClone() const override { + return new TMatchRecognizeColumnAccessNode(Pos, Var, Column); + } + +private: + void BuildLookup(TString varKeyName) { + Add( + "Member", + Y( + "Lookup", + Y( + "ToIndexDict", + BuildAtom(Pos, VarDataName) + ), + BuildAtom(Pos, std::move(varKeyName)) + ), + Q(Column) + ); + } + +private: + TString Var; + TString Column; +}; + +class TMatchRecognizeDefineAggregate final : public TAstListNode { +public: + TMatchRecognizeDefineAggregate(TPosition pos, TString name, TVector<TNodePtr> args) + : TAstListNode(pos) + , Name(std::move(name)) + , Args(std::move(args)) { + } + + bool DoInit(TContext& ctx, ISource* src) override { + Y_DEBUG_ABORT_UNLESS(ctx.GetColumnReferenceState() == EColumnRefState::MatchRecognizeDefine); + TColumnRefScope scope(ctx, EColumnRefState::MatchRecognizeDefineAggregate, false, ctx.GetMatchRecognizeDefineVar()); + if (Args.size() != 1) { + ctx.Error() << "Exactly one argument is required in MATCH_RECOGNIZE navigation function"; + return false; + } + const auto arg = Args[0]; + if (!arg->Init(ctx, src)) { + return false; + } + + const auto body = [&]() -> TNodePtr { + if ("first" == Name) { + return Y("Member", Y("Head", "item"), Q("From")); + } else if ("last" == Name) { + return Y("Member", Y("Last", "item"), Q("To")); + } else { + ctx.Error() << "Unknown row pattern navigation function: " << Name; + return {}; + } + }(); + if (!body) { + return false; + } + Add("Apply", BuildLambda(Pos, Y("index"), arg), body); + return true; + } -} //namespace { + TNodePtr DoClone() const override { + return new TMatchRecognizeDefineAggregate(Pos, Name, Args); + } -class TMatchRecognize: public TAstListNode { +private: + TString Name; + TVector<TNodePtr> Args; +}; + +class TMatchRecognizeVarAccessNode final : public INode { +public: + TMatchRecognizeVarAccessNode(TPosition pos, TNodePtr arg) + : INode(pos) + , Arg(std::move(arg)) { + } + + [[nodiscard]] const TString& GetVar() const noexcept { + return Var; + } + + TAggregationPtr GetAggregation() const override { + return Arg->GetAggregation(); + } + + bool DoInit(TContext& ctx, ISource* src) override { + if (!Arg->Init(ctx, src)) { + return false; + } + Var = ctx.ExtractMatchRecognizeAggrVar(); + Expr = [&]() -> TNodePtr { + switch (ctx.GetColumnReferenceState()) { + case EColumnRefState::MatchRecognizeMeasures: + return Arg; + case EColumnRefState::MatchRecognizeDefine: + return Y( + "Apply", + BuildLambda(Pos, Y("item"), Arg), + Y( + "Member", + BuildAtom(ctx.Pos(), VarMatchedVarsName), + Q(Var) + ) + ); + default: + Y_ABORT("Unexpected column reference state"); + } + }(); + return Expr->Init(ctx, src); + } + + TNodePtr DoClone() const override { + return new TMatchRecognizeVarAccessNode(Pos, Arg); + } + + TAstNode* Translate(TContext& ctx) const override { + return Expr->Translate(ctx); + } + +private: + TString Var; + TNodePtr Arg; + TNodePtr Expr; +}; + +class TMatchRecognize final : public TAstListNode { public: TMatchRecognize( - TPosition pos, - ISource* source, - const TString& inputTable, - std::pair<TPosition, TVector<TNamedFunction>>&& partitioners, - std::pair<TPosition, TVector<TSortSpecificationPtr>>&& sortSpecs, - std::pair<TPosition, TVector<TNamedFunction>>&& measures, - std::pair<TPosition, NYql::NMatchRecognize::ERowsPerMatch>&& rowsPerMatch, - std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo>&& skipTo, - std::pair<TPosition, NYql::NMatchRecognize::TRowPattern>&& pattern, - std::pair<TPosition, TNodePtr>&& subset, - std::pair<TPosition, TVector<TNamedFunction>>&& definitions - ): TAstListNode(pos, {BuildAtom(pos, "block")}) - { - Add(BuildBlockStatements( - pos, - source, - inputTable, - std::move(partitioners), - std::move(sortSpecs), - std::move(measures), - std::move(rowsPerMatch), - std::move(skipTo), - std::move(pattern), - std::move(subset), - std::move(definitions) - )); + TPosition pos, + TString label, + TNodePtr partitionKeySelector, + TNodePtr partitionColumns, + TVector<TSortSpecificationPtr> sortSpecs, + TVector<TNamedFunction> measures, + TNodePtr rowsPerMatch, + TNodePtr skipTo, + TNodePtr pattern, + TNodePtr patternVars, + TNodePtr subset, + TVector<TNamedFunction> definitions) + : TAstListNode(pos) + , Label(std::move(label)) + , PartitionKeySelector(std::move(partitionKeySelector)) + , PartitionColumns(std::move(partitionColumns)) + , SortSpecs(std::move(sortSpecs)) + , Measures(std::move(measures)) + , RowsPerMatch(std::move(rowsPerMatch)) + , SkipTo(std::move(skipTo)) + , Pattern(std::move(pattern)) + , PatternVars(std::move(patternVars)) + , Subset(std::move(subset)) + , Definitions(std::move(definitions)) { } + private: - TMatchRecognize(const TMatchRecognize& other) - : TAstListNode(other.Pos) - { - Nodes = CloneContainer(other.Nodes); - } - - TNodePtr BuildBlockStatements( - TPosition pos, - ISource* source, - const TString& inputTable, - std::pair<TPosition, TVector<TNamedFunction>>&& partitioners, - std::pair<TPosition, TVector<TSortSpecificationPtr>>&& sortSpecs, - std::pair<TPosition, TVector<TNamedFunction>>&& measures, - std::pair<TPosition, NYql::NMatchRecognize::ERowsPerMatch>&& rowsPerMatch, - std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo>&& skipTo, - std::pair<TPosition, NYql::NMatchRecognize::TRowPattern>&& pattern, - std::pair<TPosition, TNodePtr>&& subset, - std::pair<TPosition, TVector<TNamedFunction>>&& definitions - ) { - Y_UNUSED(pos); - - auto inputRowType = Y("ListItemType",Y("TypeOf", inputTable)); - - auto patternNode = Pattern(pattern.first, pattern.second); - - auto partitionColumns = Y(); - for (const auto& p: partitioners.second){ - partitionColumns->Add(BuildQuotedAtom(p.callable->GetPos(), p.name)); - } - partitionColumns = Q(partitionColumns); - auto partitionKeySelector = Y(); - for (const auto& p: partitioners.second){ - partitionKeySelector->Add(p.callable); - } - partitionKeySelector = BuildLambda(partitioners.first, Y("row"), Q(partitionKeySelector)); + bool DoInit(TContext& ctx, ISource* src) override { + auto inputRowType = Y("ListItemType", Y("TypeOf", Label)); + + if (!PartitionKeySelector->Init(ctx, src)) { + return false; + } + if (!PartitionColumns->Init(ctx, src)) { + return false; + } + + const auto sortTraits = SortSpecs.empty() ? Y("Void") : src->BuildSortSpec(SortSpecs, Label, true, false); + if (!sortTraits->Init(ctx, src)) { + return false; + } auto measureNames = Y(); - for (const auto& m: measures.second){ - measureNames->Add(BuildQuotedAtom(m.callable->GetPos(), m.name)); + for (const auto& m: Measures) { + measureNames->Add(BuildQuotedAtom(m.Callable->GetPos(), m.Name)); } - TNodePtr measuresNode = Y("MatchRecognizeMeasures", inputRowType, patternNode, Q(measureNames)); - for (const auto& m: measures.second){ - measuresNode->Add(BuildLambda(m.callable->GetPos(), Y(VarDataName, VarMatchedVarsName), m.callable)); + auto measureVars = Y(); + for (const auto& m: Measures) { + TColumnRefScope scope(ctx, EColumnRefState::MatchRecognizeMeasures); + if (!m.Callable->Init(ctx, src)) { + return false; + } + const auto varAccess = dynamic_cast<TMatchRecognizeVarAccessNode*>(m.Callable.Get()); + auto var = varAccess ? varAccess->GetVar() : ""; + measureVars->Add(BuildQuotedAtom(m.Callable->GetPos(), std::move(var))); } - auto defineNames = Y(); - for (const auto& d: definitions.second) { - defineNames->Add(BuildQuotedAtom(d.callable->GetPos(), d.name)); + auto measuresNode = Y("MatchRecognizeMeasuresAggregates", inputRowType, Q(PatternVars), Q(measureNames), Q(measureVars)); + for (const auto& m: Measures) { + auto aggr = m.Callable->GetAggregation(); + if (!aggr) { + // TODO(YQL-16508): support aggregations inside expressions + // ctx.Error(m.Callable->GetPos()) << "Cannot use aggregations inside expression"; + // return false; + measuresNode->Add(m.Callable); + } else { + const auto [traits, result] = aggr->AggregationTraits(Y("TypeOf", Label), false, false, false, ctx); + if (!result) { + return false; + } + measuresNode->Add(traits); + } + } + + if (!RowsPerMatch->Init(ctx, src)) { + return false; } - TNodePtr defineNode = Y("MatchRecognizeDefines", inputRowType, patternNode, Q(defineNames)); - for (const auto& d: definitions.second) { - defineNode->Add(BuildLambda(d.callable->GetPos(), Y(VarDataName, VarMatchedVarsName, VarLastRowIndexName), d.callable)); + if (!SkipTo->Init(ctx, src)) { + return false; + } + + if (!Pattern->Init(ctx, src)) { + return false; + } + + if (!PatternVars->Init(ctx, src)) { + return false; + } + + auto defineNames = Y(); + for (const auto& d: Definitions) { + defineNames->Add(BuildQuotedAtom(d.Callable->GetPos(), d.Name)); + } + auto defineNode = Y("MatchRecognizeDefines", inputRowType, Q(PatternVars), Q(defineNames)); + for (const auto& d: Definitions) { + TColumnRefScope scope(ctx, EColumnRefState::MatchRecognizeDefine, true, d.Name); + if (!d.Callable->Init(ctx, src)) { + return false; + } + defineNode->Add(BuildLambda(d.Callable->GetPos(), Y(VarDataName, VarMatchedVarsName, VarLastRowIndexName), d.Callable)); } - return Q(Y( - Y("let", "input", inputTable), - Y("let", "partitionKeySelector", partitionKeySelector), - Y("let", "partitionColumns", partitionColumns), - Y("let", "sortTraits", sortSpecs.second.empty()? Y("Void") : source->BuildSortSpec(sortSpecs.second, inputTable, true, false)), + Add( + "block", + Q(Y( + Y("let", "input", Label), + Y("let", "partitionKeySelector", PartitionKeySelector), + Y("let", "partitionColumns", PartitionColumns), + Y("let", "sortTraits", sortTraits), Y("let", "measures", measuresNode), - Y("let", "rowsPerMatch", BuildQuotedAtom(rowsPerMatch.first, "RowsPerMatch_" + ToString(rowsPerMatch.second))), - Y("let", "skipTo", BuildTuple(skipTo.first, {Q("AfterMatchSkip_" + ToString(skipTo.second.To)), Q(ToString(skipTo.second.Var))})), - Y("let", "pattern", patternNode), - Y("let", "subset", subset.second ? subset.second : Q("")), + Y("let", "rowsPerMatch", RowsPerMatch), + Y("let", "skipTo", SkipTo), + Y("let", "pattern", Pattern), + Y("let", "subset", Subset ? Subset : Q("")), Y("let", "define", defineNode), Y("let", "res", Y("MatchRecognize", "input", @@ -122,133 +313,76 @@ private: ) )), Y("return", "res") - )); - } - - TPtr PatternFactor(const TPosition& pos, const NYql::NMatchRecognize::TRowPatternFactor& factor) { - return BuildTuple(pos, { - factor.Primary.index() == 0 ? - BuildQuotedAtom(pos, std::get<0>(factor.Primary)) : - Pattern(pos, std::get<1>(factor.Primary)), - BuildQuotedAtom(pos, ToString(factor.QuantityMin)), - BuildQuotedAtom(pos, ToString(factor.QuantityMax)), - BuildQuotedAtom(pos, ToString(factor.Greedy)), - BuildQuotedAtom(pos, ToString(factor.Output)), - BuildQuotedAtom(pos, ToString(factor.Unused)) - }); - } - - - TPtr PatternTerm(const TPosition& pos, const NYql::NMatchRecognize::TRowPatternTerm& term) { - auto factors = Y(); - for (const auto& f: term) - factors->Add(PatternFactor(pos, f)); - return Q(std::move(factors)); + )) + ); + return true; } - TPtr Pattern(const TPosition& pos, const NYql::NMatchRecognize::TRowPattern& pattern) { - TNodePtr patternNode = Y("MatchRecognizePattern"); - for (const auto& t: pattern) { - patternNode->Add(PatternTerm(pos, t)); - } - return patternNode; + TNodePtr DoClone() const override { + return new TMatchRecognize( + Pos, + Label, + PartitionKeySelector, + PartitionColumns, + SortSpecs, + Measures, + RowsPerMatch, + SkipTo, + Pattern, + PatternVars, + Subset, + Definitions + ); } - TPtr DoClone() const final{ - return new TMatchRecognize(*this); - } +private: + TString Label; + TNodePtr PartitionKeySelector; + TNodePtr PartitionColumns; + TVector<TSortSpecificationPtr> SortSpecs; + TVector<TNamedFunction> Measures; + TNodePtr RowsPerMatch; + TNodePtr SkipTo; + TNodePtr Pattern; + TNodePtr PatternVars; + TNodePtr Subset; + TVector<TNamedFunction> Definitions; }; -TNodePtr TMatchRecognizeBuilder::Build(TContext& ctx, TString&& inputTable, ISource* source){ +} // anonymous namespace + +TNodePtr TMatchRecognizeBuilder::Build(TContext& ctx, TString label, ISource* src) { TNodePtr node = new TMatchRecognize( - Pos, - source, - std::move(inputTable), - std::move(Partitioners), - std::move(SortSpecs), - std::move(Measures), - std::move(RowsPerMatch), - std::move(SkipTo), - std::move(Pattern), - std::move(Subset), - std::move(Definitions) + Pos, + std::move(label), + std::move(PartitionKeySelector), + std::move(PartitionColumns), + std::move(SortSpecs), + std::move(Measures), + std::move(RowsPerMatch), + std::move(SkipTo), + std::move(Pattern), + std::move(PatternVars), + std::move(Subset), + std::move(Definitions) ); - if (!node->Init(ctx, source)) - return nullptr; + if (!node->Init(ctx, src)) { + return {}; + } return node; } -namespace { -const auto DefaultNavigatingFunction = "MatchRecognizeDefaultNavigating"; +TNodePtr BuildMatchRecognizeColumnAccess(TPosition pos, TString var, TString column) { + return MakeIntrusive<TMatchRecognizeColumnAccessNode>(pos, std::move(var), std::move(column)); } -bool TMatchRecognizeVarAccessNode::DoInit(TContext& ctx, ISource* src) { - //If referenced var is the var that is currently being defined - //then it's a reference to the last row in a partition - Node = new TMatchRecognizeNavigate(ctx.Pos(), DefaultNavigatingFunction, TVector<TNodePtr>{this->Clone()}); - return Node->Init(ctx, src); +TNodePtr BuildMatchRecognizeDefineAggregate(TPosition pos, TString name, TVector<TNodePtr> args) { + const auto result = MakeIntrusive<TMatchRecognizeDefineAggregate>(pos, std::move(name), std::move(args)); + return BuildMatchRecognizeVarAccess(pos, std::move(result)); } -bool TMatchRecognizeNavigate::DoInit(TContext& ctx, ISource* src) { - Y_UNUSED(src); - if (Args.size() != 1) { - ctx.Error(Pos) << "Exactly one argument is required in MATCH_RECOGNIZE navigation function"; - return false; - } - const auto varColumn = dynamic_cast<TMatchRecognizeVarAccessNode *>(Args[0].Get()); - if (not varColumn) { - ctx.Error(Pos) << "Row pattern navigation operations are applicable to row pattern variable only"; - return false; - } - const auto varData = BuildAtom(ctx.Pos(), VarDataName); - const auto varMatchedVars = BuildAtom(ctx.Pos(), VarMatchedVarsName); - const auto varLastRowIndex = BuildAtom(ctx.Pos(), VarLastRowIndexName); - - const auto matchedRanges = Y("Member", varMatchedVars, Q(varColumn->GetVar())); - TNodePtr navigatedRowIndex; - if (DefaultNavigatingFunction == Name) { - if (not varColumn->IsTheSameVar()) { - ctx.Error(Pos) << "Row pattern navigation function is required"; - return false; - } - navigatedRowIndex = varLastRowIndex; - } - else if ("PREV" == Name) { - if (not varColumn->IsTheSameVar()) { - ctx.Error(Pos) << "PREV relative to matched vars is not implemented yet"; - return false; - } - navigatedRowIndex = Y( - "-", - varLastRowIndex, - Y("Uint64", Q("1")) - ); - } else if ("FIRST" == Name) { - navigatedRowIndex = Y( - "Member", - Y("Head", matchedRanges), - Q("From") - ); - } else if ("LAST" == Name) { - navigatedRowIndex = Y( - "Member", - Y("Last", matchedRanges), - Q("To") - ); - } else { - ctx.Error(Pos) << "Internal logic error"; - return false; - } - Add("Member"); - Add( - Y( - "Lookup", - Y("ToIndexDict", varData), - navigatedRowIndex - ) - ), - Add(Q(varColumn->GetColumn())); - return true; +TNodePtr BuildMatchRecognizeVarAccess(TPosition pos, TNodePtr extractor) { + return MakeIntrusive<TMatchRecognizeVarAccessNode>(pos, std::move(extractor)); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/match_recognize.h b/yql/essentials/sql/v1/match_recognize.h index 4b0e98b9b71..818bd6e6002 100644 --- a/yql/essentials/sql/v1/match_recognize.h +++ b/yql/essentials/sql/v1/match_recognize.h @@ -1,125 +1,64 @@ #pragma once + #include "node.h" + #include <yql/essentials/core/sql_types/match_recognize.h> #include <util/generic/ptr.h> namespace NSQLTranslationV1 { struct TNamedFunction { - TNodePtr callable; //Callable with some free args - TString name; + TNodePtr Callable; + TString Name; }; class TMatchRecognizeBuilder: public TSimpleRefCount<TMatchRecognizeBuilder> { public: TMatchRecognizeBuilder( - TPosition clausePos, - std::pair<TPosition, TVector<TNamedFunction>>&& partitioners, - std::pair<TPosition, TVector<TSortSpecificationPtr>>&& sortSpecs, - std::pair<TPosition, TVector<TNamedFunction>>&& measures, - std::pair<TPosition, NYql::NMatchRecognize::ERowsPerMatch>&& rowsPerMatch, - std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo>&& skipTo, - std::pair<TPosition, NYql::NMatchRecognize::TRowPattern>&& pattern, - std::pair<TPosition, TNodePtr>&& subset, - std::pair<TPosition, TVector<TNamedFunction>>&& definitions - ) - : Pos(clausePos) - , Partitioners(std::move(partitioners)) - , SortSpecs(std::move(sortSpecs)) - , Measures(std::move(measures)) - , RowsPerMatch(std::move(rowsPerMatch)) - , SkipTo(std::move(skipTo)) - , Pattern(std::move(pattern)) - , Subset(std::move(subset)) - , Definitions(definitions) - + TPosition pos, + TNodePtr partitionKeySelector, + TNodePtr partitionColumns, + TVector<TSortSpecificationPtr> sortSpecs, + TVector<TNamedFunction> measures, + TNodePtr rowsPerMatch, + TNodePtr skipTo, + TNodePtr pattern, + TNodePtr patternVars, + TNodePtr subset, + TVector<TNamedFunction> definitions) + : Pos(pos) + , PartitionKeySelector(std::move(partitionKeySelector)) + , PartitionColumns(std::move(partitionColumns)) + , SortSpecs(std::move(sortSpecs)) + , Measures(std::move(measures)) + , RowsPerMatch(std::move(rowsPerMatch)) + , SkipTo(std::move(skipTo)) + , Pattern(std::move(pattern)) + , PatternVars(std::move(patternVars)) + , Subset(std::move(subset)) + , Definitions(std::move(definitions)) {} - TNodePtr Build(TContext& ctx, TString&& inputTable, ISource* source); -private: - TPosition Pos; - std::pair<TPosition, TVector<TNamedFunction>> Partitioners; - std::pair<TPosition, TVector<TSortSpecificationPtr>> SortSpecs; - std::pair<TPosition, TVector<TNamedFunction>> Measures; - std::pair<TPosition, NYql::NMatchRecognize::ERowsPerMatch> RowsPerMatch; - std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo> SkipTo; - std::pair<TPosition, NYql::NMatchRecognize::TRowPattern> Pattern; - std::pair<TPosition, TNodePtr> Subset; - std::pair<TPosition, TVector<TNamedFunction>> Definitions; -}; - -using TMatchRecognizeBuilderPtr=TIntrusivePtr<TMatchRecognizeBuilder> ; - -class TMatchRecognizeVarAccessNode: public INode { -public: - TMatchRecognizeVarAccessNode(TPosition pos, const TString& var, const TString& column, bool theSameVar) - : INode(pos) - , Var(var) - , TheSameVar(theSameVar) - , Column(column) - { - } - - TString GetVar() const { - return Var; - } - - bool IsTheSameVar() const { - return TheSameVar; - } - - TString GetColumn() const { - return Column; - } - bool DoInit(TContext& ctx, ISource* src) override; - - TAstNode* Translate(TContext& ctx) const override { - return Node->Translate(ctx); - } - - TPtr DoClone() const override { - YQL_ENSURE(!Node, "TMatchRecognizeVarAccessNode::Clone: Node must not be initialized"); - auto copy = new TMatchRecognizeVarAccessNode(Pos, Var, Column, TheSameVar); - return copy; - } - -protected: - void DoUpdateState() const override { - YQL_ENSURE(Node); - } - - void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final { - Y_DEBUG_ABORT_UNLESS(Node); - Node->VisitTree(func, visited); - } + TNodePtr Build(TContext& ctx, TString label, ISource* source); private: - TNodePtr Node; - const TString Var; - const bool TheSameVar; //reference the same var as being defined by this expression; - const TString Column; + TPosition Pos; + TNodePtr PartitionKeySelector; + TNodePtr PartitionColumns; + TVector<TSortSpecificationPtr> SortSpecs; + TVector<TNamedFunction> Measures; + TNodePtr RowsPerMatch; + TNodePtr SkipTo; + TNodePtr Pattern; + TNodePtr PatternVars; + TNodePtr Subset; + TVector<TNamedFunction> Definitions; }; -class TMatchRecognizeNavigate: public TAstListNode { -public: - TMatchRecognizeNavigate(TPosition pos, const TString& name, const TVector<TNodePtr>& args) - : TAstListNode(pos) - , Name(name) - , Args(args) - { - } - -private: - TNodePtr DoClone() const override { - return new TMatchRecognizeNavigate(GetPos(), Name, CloneContainer(Args)); - } - - bool DoInit(TContext& ctx, ISource* src) override; +using TMatchRecognizeBuilderPtr = TIntrusivePtr<TMatchRecognizeBuilder>; -private: - const TString Name; - const TVector<TNodePtr> Args; -}; +TNodePtr BuildMatchRecognizeColumnAccess(TPosition pos, TString var, TString column); +TNodePtr BuildMatchRecognizeDefineAggregate(TPosition pos, TString name, TVector<TNodePtr> args); +TNodePtr BuildMatchRecognizeVarAccess(TPosition pos, TNodePtr extractor); } // namespace NSQLTranslationV1 - diff --git a/yql/essentials/sql/v1/node.cpp b/yql/essentials/sql/v1/node.cpp index 2339fa894e7..eea9069abf3 100644 --- a/yql/essentials/sql/v1/node.cpp +++ b/yql/essentials/sql/v1/node.cpp @@ -706,6 +706,14 @@ TAstNode* TAstDirectNode::Translate(TContext& ctx) const { return Node; } +TNodePtr BuildList(TPosition pos, TVector<TNodePtr> nodes) { + return new TAstListNodeImpl(pos, std::move(nodes)); +} + +TNodePtr BuildQuote(TPosition pos, TNodePtr expr) { + return BuildList(pos, {BuildAtom(pos, "quote", TNodeFlags::Default), expr}); +} + TNodePtr BuildAtom(TPosition pos, const TString& content, ui32 flags, bool isOptionalArg) { return new TAstAtomNodeImpl(pos, content, flags, isOptionalArg); } @@ -2669,10 +2677,6 @@ TNodePtr BuildAccess(TPosition pos, const TVector<INode::TIdPart>& ids, bool isL return new TAccessNode(pos, ids, isLookup); } -TNodePtr BuildMatchRecognizeVarAccess(TPosition pos, const TString& var, const TString& column, bool theSameVar) { - return new TMatchRecognizeVarAccessNode(pos, var, column, theSameVar); -} - void WarnIfAliasFromSelectIsUsedInGroupBy(TContext& ctx, const TVector<TNodePtr>& selectTerms, const TVector<TNodePtr>& groupByTerms, const TVector<TNodePtr>& groupByExprTerms) { diff --git a/yql/essentials/sql/v1/node.h b/yql/essentials/sql/v1/node.h index c9402a2f97b..c197f7ac900 100644 --- a/yql/essentials/sql/v1/node.h +++ b/yql/essentials/sql/v1/node.h @@ -1437,6 +1437,8 @@ namespace NSQLTranslationV1 { TString TypeByAlias(const TString& alias, bool normalize = true); + TNodePtr BuildList(TPosition pos, TVector<TNodePtr> nodes = {}); + TNodePtr BuildQuote(TPosition pos, TNodePtr expr); TNodePtr BuildAtom(TPosition pos, const TString& content, ui32 flags = NYql::TNodeFlags::ArbitraryContent, bool isOptionalArg = false); TNodePtr BuildQuotedAtom(TPosition pos, const TString& content, ui32 flags = NYql::TNodeFlags::ArbitraryContent); @@ -1471,7 +1473,6 @@ namespace NSQLTranslationV1 { TNodePtr BuildColumn(TPosition pos, const TDeferredAtom& column, const TString& source = TString()); TNodePtr BuildColumnOrType(TPosition pos, const TString& column = TString()); TNodePtr BuildAccess(TPosition pos, const TVector<INode::TIdPart>& ids, bool isLookup); - TNodePtr BuildMatchRecognizeVarAccess(TPosition pos, const TString& var, const TString& column, bool theSameVar); TNodePtr BuildBind(TPosition pos, const TString& module, const TString& alias); TNodePtr BuildLambda(TPosition pos, TNodePtr params, TNodePtr body, const TString& resName = TString()); TNodePtr BuildLambda(TPosition pos, TNodePtr params, const TVector<TNodePtr>& bodies); diff --git a/yql/essentials/sql/v1/source.cpp b/yql/essentials/sql/v1/source.cpp index 4231a9d370a..c2fb13b103b 100644 --- a/yql/essentials/sql/v1/source.cpp +++ b/yql/essentials/sql/v1/source.cpp @@ -288,7 +288,7 @@ inline TNodePtr ISource::AliasOrColumn(const TNodePtr& node, bool withSource) { bool ISource::AddAggregationOverWindow(TContext& ctx, const TString& windowName, TAggregationPtr func) { if (ctx.DistinctOverWindow) { - YQL_ENSURE(func->IsOverWindow() || func->IsOverWindowDistinct()); + YQL_ENSURE(func->IsOverWindow() || func->IsOverWindowDistinct()); } else { YQL_ENSURE(func->IsOverWindow()); if (func->IsDistinct()) { diff --git a/yql/essentials/sql/v1/sql_expression.cpp b/yql/essentials/sql/v1/sql_expression.cpp index c2074cb50d9..be401949c7c 100644 --- a/yql/essentials/sql/v1/sql_expression.cpp +++ b/yql/essentials/sql/v1/sql_expression.cpp @@ -860,24 +860,7 @@ TNodePtr TSqlExpression::JsonApiExpr(const TRule_json_api_expr& node) { return result; } -TNodePtr MatchRecognizeVarAccess(TTranslation& ctx, const TString& var, const TRule_an_id_or_type& suffix, bool theSameVar) { - switch (suffix.GetAltCase()) { - case TRule_an_id_or_type::kAltAnIdOrType1: - break; - case TRule_an_id_or_type::kAltAnIdOrType2: - break; - case TRule_an_id_or_type::ALT_NOT_SET: - break; - } - const auto& column = Id( - suffix.GetAlt_an_id_or_type1() - .GetRule_id_or_type1().GetAlt_id_or_type1().GetRule_id1(), - ctx - ); - return BuildMatchRecognizeVarAccess(TPosition{}, var, column, theSameVar); -} - -TNodePtr TSqlExpression::RowPatternVarAccess(const TString& alias, const TRule_unary_subexpr_suffix_TBlock1_TBlock1_TAlt3_TBlock2 block) { +TNodePtr TSqlExpression::RowPatternVarAccess(TString var, const TRule_unary_subexpr_suffix_TBlock1_TBlock1_TAlt3_TBlock2 block) { switch (block.GetAltCase()) { case TRule_unary_subexpr_suffix_TBlock1_TBlock1_TAlt3_TBlock2::kAlt1: break; @@ -888,13 +871,10 @@ TNodePtr TSqlExpression::RowPatternVarAccess(const TString& alias, const TRule_u case TRule_an_id_or_type::kAltAnIdOrType1: { const auto &idOrType = block.GetAlt3().GetRule_an_id_or_type1().GetAlt_an_id_or_type1().GetRule_id_or_type1(); switch(idOrType.GetAltCase()) { - case TRule_id_or_type::kAltIdOrType1: - return BuildMatchRecognizeVarAccess( - Ctx.Pos(), - alias, - Id(idOrType.GetAlt_id_or_type1().GetRule_id1(), *this), - Ctx.GetMatchRecognizeDefineVar() == alias - ); + case TRule_id_or_type::kAltIdOrType1: { + const auto column = Id(idOrType.GetAlt_id_or_type1().GetRule_id1(), *this); + return BuildMatchRecognizeColumnAccess(Ctx.Pos(), std::move(var), std::move(column)); + } case TRule_id_or_type::kAltIdOrType2: break; case TRule_id_or_type::ALT_NOT_SET: @@ -911,7 +891,7 @@ TNodePtr TSqlExpression::RowPatternVarAccess(const TString& alias, const TRule_u case TRule_unary_subexpr_suffix_TBlock1_TBlock1_TAlt3_TBlock2::ALT_NOT_SET: Y_ABORT("You should change implementation according to grammar changes"); } - return TNodePtr{}; + return {}; } template<typename TUnaryCasualExprRule> @@ -991,13 +971,12 @@ TNodePtr TSqlExpression::UnaryCasualExpr(const TUnaryCasualExprRule& node, const case TRule_unary_subexpr_suffix::TBlock1::TBlock1::kAlt3: { // In case of MATCH_RECOGNIZE lambdas // X.Y is treated as Var.Column access - if (isColumnRef && EColumnRefState::MatchRecognize == Ctx.GetColumnReferenceState()) { - if (auto rowPatternVarAccess = RowPatternVarAccess( - name, - b.GetAlt3().GetBlock2()) - ) { - return rowPatternVarAccess; - } + if (isColumnRef && ( + EColumnRefState::MatchRecognizeMeasures == Ctx.GetColumnReferenceState() || + EColumnRefState::MatchRecognizeDefine == Ctx.GetColumnReferenceState() || + EColumnRefState::MatchRecognizeDefineAggregate == Ctx.GetColumnReferenceState() + )) { + return RowPatternVarAccess(std::move(name), b.GetAlt3().GetBlock2()); } break; } diff --git a/yql/essentials/sql/v1/sql_expression.h b/yql/essentials/sql/v1/sql_expression.h index 4f9100722ca..adcf44b11b6 100644 --- a/yql/essentials/sql/v1/sql_expression.h +++ b/yql/essentials/sql/v1/sql_expression.h @@ -109,7 +109,7 @@ private: TNodePtr BinOperList(const TString& opName, TVector<TNodePtr>::const_iterator begin, TVector<TNodePtr>::const_iterator end) const; - TNodePtr RowPatternVarAccess(const TString& alias, const TRule_unary_subexpr_suffix_TBlock1_TBlock1_TAlt3_TBlock2 block); + TNodePtr RowPatternVarAccess(TString var, const TRule_unary_subexpr_suffix_TBlock1_TBlock1_TAlt3_TBlock2 block); struct TCaseBranch { TNodePtr Pred; diff --git a/yql/essentials/sql/v1/sql_match_recognize.cpp b/yql/essentials/sql/v1/sql_match_recognize.cpp index 41415b7f239..6fcd56f4d6e 100644 --- a/yql/essentials/sql/v1/sql_match_recognize.cpp +++ b/yql/essentials/sql/v1/sql_match_recognize.cpp @@ -1,367 +1,467 @@ #include "sql_match_recognize.h" + #include "node.h" #include "sql_expression.h" + #include <yql/essentials/core/sql_types/match_recognize.h> -#include <algorithm> namespace NSQLTranslationV1 { -using namespace NSQLv1Generated; - -namespace { - -TPosition TokenPosition(const TToken& token){ - return TPosition{token.GetColumn(), token.GetLine()}; -} - -TString PatternVar(const TRule_row_pattern_variable_name& node, TSqlMatchRecognizeClause& ctx){ - return Id(node.GetRule_identifier1(), ctx); -} - -} //namespace +TSqlMatchRecognizeClause::TSqlMatchRecognizeClause(TContext& ctx, NSQLTranslation::ESqlMode mode) : TSqlTranslation(ctx, mode) {} TMatchRecognizeBuilderPtr TSqlMatchRecognizeClause::CreateBuilder(const NSQLv1Generated::TRule_row_pattern_recognition_clause &matchRecognizeClause) { - TPosition pos(matchRecognizeClause.GetToken1().GetColumn(), matchRecognizeClause.GetToken1().GetLine()); + auto pos = GetPos(matchRecognizeClause.GetToken1()); if (!Ctx.FeatureR010) { Ctx.Error(pos, TIssuesIds::CORE) << "Unexpected MATCH_RECOGNIZE"; return {}; } - TVector<TNamedFunction> partitioners; - TPosition partitionsPos = pos; - if (matchRecognizeClause.HasBlock3()) { - const auto& partitionClause = matchRecognizeClause.GetBlock3().GetRule_window_partition_clause1(); - partitionsPos = TokenPosition(partitionClause.GetToken1()); - partitioners = ParsePartitionBy(partitionClause); - if (!partitioners) - return {}; - } - TVector<TSortSpecificationPtr> sortSpecs; - TPosition orderByPos = pos; - if (matchRecognizeClause.HasBlock4()) { - const auto& orderByClause = matchRecognizeClause.GetBlock4().GetRule_order_by_clause1(); - orderByPos = TokenPosition(orderByClause.GetToken1()); - if (!OrderByClause(orderByClause, sortSpecs)) { - return {}; - } - } - TPosition measuresPos = pos; - TVector<TNamedFunction> measures; - if (matchRecognizeClause.HasBlock5()) { - const auto& measuresClause = matchRecognizeClause.GetBlock5().GetRule_row_pattern_measures1(); - measuresPos = TokenPosition(measuresClause.GetToken1()); - measures = ParseMeasures(measuresClause.GetRule_row_pattern_measure_list2()); + auto [partitionKeySelector, partitionColumns] = ParsePartitionBy( + pos, + matchRecognizeClause.HasBlock3() + ? std::addressof(matchRecognizeClause.GetBlock3().GetRule_window_partition_clause1()) + : nullptr + ); + + auto sortSpecs = ParseOrderBy( + matchRecognizeClause.HasBlock4() + ? std::addressof(matchRecognizeClause.GetBlock4().GetRule_order_by_clause1()) + : nullptr + ); + if (!sortSpecs) { + return {}; } - auto rowsPerMatch = std::pair {pos, NYql::NMatchRecognize::ERowsPerMatch::OneRow}; - if (matchRecognizeClause.HasBlock6()) { - rowsPerMatch = ParseRowsPerMatch(matchRecognizeClause.GetBlock6().GetRule_row_pattern_rows_per_match1()); + auto measures = ParseMeasures( + matchRecognizeClause.HasBlock5() + ? std::addressof(matchRecognizeClause.GetBlock5().GetRule_row_pattern_measures1().GetRule_row_pattern_measure_list2()) + : nullptr + ); + + auto rowsPerMatch = ParseRowsPerMatch( + pos, + matchRecognizeClause.HasBlock6() + ? std::addressof(matchRecognizeClause.GetBlock6().GetRule_row_pattern_rows_per_match1()) + : nullptr + ); + if (!rowsPerMatch) { + return {}; } const auto& commonSyntax = matchRecognizeClause.GetRule_row_pattern_common_syntax7(); - if (commonSyntax.HasBlock2()) { const auto& initialOrSeek = commonSyntax.GetBlock2().GetRule_row_pattern_initial_or_seek1(); - Ctx.Error(TokenPosition(initialOrSeek.GetToken1())) << "InitialOrSeek subclause is not allowed in FROM clause"; + Ctx.Error(GetPos(initialOrSeek.GetToken1())) << "InitialOrSeek subclause is not allowed in FROM clause"; return {}; } - auto pattern = ParsePattern(commonSyntax.GetRule_row_pattern5()); - const auto& patternPos = TokenPosition(commonSyntax.token3()); - - //this block is located before pattern block in grammar, - // but depends on it, so it is processed after pattern block - std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo> skipTo { - pos, - NYql::NMatchRecognize::TAfterMatchSkipTo{ - NYql::NMatchRecognize::EAfterMatchSkipTo::PastLastRow, - TString() - } - }; - if (commonSyntax.HasBlock1()){ - skipTo = ParseAfterMatchSkipTo(commonSyntax.GetBlock1().GetRule_row_pattern_skip_to3()); - const auto varRequired = - NYql::NMatchRecognize::EAfterMatchSkipTo::ToFirst == skipTo.second.To || - NYql::NMatchRecognize::EAfterMatchSkipTo::ToLast == skipTo.second.To || - NYql::NMatchRecognize::EAfterMatchSkipTo::To == skipTo.second.To; - if (varRequired) { - const auto& allVars = NYql::NMatchRecognize::GetPatternVars(pattern); - if (allVars.find(skipTo.second.Var) == allVars.cend()) { - Ctx.Error(skipTo.first) << "Unknown pattern variable in AFTER MATCH"; - return {}; - } - } + PatternVarNames.clear(); + PatternVars = BuildList(pos); + auto pattern = ParsePattern(pos, commonSyntax.GetRule_row_pattern5(), 0, true); + if (!pattern) { + return {}; } + auto skipTo = ParseAfterMatchSkipTo( + pos, + commonSyntax.HasBlock1() + ? std::addressof(commonSyntax.GetBlock1().GetRule_row_pattern_skip_to3()) + : nullptr + ); + if (!skipTo) { + return {}; + } - TNodePtr subset; - TPosition subsetPos = pos; - if (commonSyntax.HasBlock7()) { - const auto& rowPatternSubset = commonSyntax.GetBlock7().GetRule_row_pattern_subset_clause1(); - subsetPos = TokenPosition(rowPatternSubset.GetToken1()); - Ctx.Error() << "SUBSET is not implemented yet"; - //TODO https://st.yandex-team.ru/YQL-16225 + auto subset = ParseSubset( + pos, + commonSyntax.HasBlock7() + ? std::addressof(commonSyntax.GetBlock7().GetRule_row_pattern_subset_clause1()) + : nullptr + ); + if (!subset) { return {}; } - const auto& definitions = ParseDefinitions(commonSyntax.GetRule_row_pattern_definition_list9()); - const auto& definitionsPos = TokenPosition(commonSyntax.GetToken8()); - const auto& rowPatternVariables = GetPatternVars(pattern); + auto definitions = ParseDefinitions(commonSyntax.GetRule_row_pattern_definition_list9()); for (const auto& [callable, name]: definitions) { - if (!rowPatternVariables.contains(name)) { + if (!PatternVarNames.contains(name)) { Ctx.Error(callable->GetPos()) << "ROW PATTERN VARIABLE " << name << " is defined, but not mentioned in the PATTERN"; return {}; } } - return new TMatchRecognizeBuilder{ + return new TMatchRecognizeBuilder( pos, - std::pair{partitionsPos, std::move(partitioners)}, - std::pair{orderByPos, std::move(sortSpecs)}, - std::pair{measuresPos, measures}, + std::move(partitionKeySelector), + std::move(partitionColumns), + std::move(*sortSpecs), + std::move(measures), std::move(rowsPerMatch), std::move(skipTo), - std::pair{patternPos, std::move(pattern)}, - std::pair{subsetPos, std::move(subset)}, - std::pair{definitionsPos, std::move(definitions)} - }; - + std::move(pattern), + std::move(PatternVars), + std::move(*subset), + std::move(definitions) + ); +} +std::tuple<TNodePtr, TNodePtr> TSqlMatchRecognizeClause::ParsePartitionBy(TPosition pos, const TRule_window_partition_clause* node) { + auto [partitionKeySelector, partitionColumns] = [&]() -> std::tuple<TNodePtr, TNodePtr> { + auto partitionKeySelector = BuildList(pos); + auto partitionColumns = BuildList(pos); + if (!node) { + return {partitionKeySelector, partitionColumns}; + } + TColumnRefScope scope(Ctx, EColumnRefState::Allow); + TVector<TNodePtr> partitionExprs; + if (!NamedExprList(node->GetRule_named_expr_list4(), partitionExprs)) { + return {partitionKeySelector, partitionColumns}; + } + for (const auto& p : partitionExprs) { + auto label = p->GetLabel(); + if (!label && p->GetColumnName()) { + label = *p->GetColumnName(); + } + partitionKeySelector->Add(p); + partitionColumns->Add(BuildQuotedAtom(p->GetPos(), label)); + } + return {partitionKeySelector, partitionColumns}; + }(); + return { + BuildLambda(pos, BuildList(pos, {BuildAtom(pos, "row")}), BuildQuote(pos, std::move(partitionKeySelector))), + BuildQuote(pos, std::move(partitionColumns)) + }; } -TVector<TNamedFunction> TSqlMatchRecognizeClause::ParsePartitionBy(const TRule_window_partition_clause& partitionClause) { - TColumnRefScope scope(Ctx, EColumnRefState::Allow); - TVector<TNodePtr> partitionExprs; - if (!NamedExprList( - partitionClause.GetRule_named_expr_list4(), - partitionExprs)) { - return {}; +TMaybe<TVector<TSortSpecificationPtr>> TSqlMatchRecognizeClause::ParseOrderBy(const TRule_order_by_clause* node) { + if (!node) { + return TVector<TSortSpecificationPtr>{}; } - TVector<TNamedFunction> partitioners; - for (const auto& p: partitionExprs) { - auto label = p->GetLabel(); - if (!label && p->GetColumnName()) { - label = *p->GetColumnName(); - } - partitioners.push_back(TNamedFunction{p, label}); + TVector<TSortSpecificationPtr> result; + if (!OrderByClause(*node, result)) { + return {}; } - return partitioners; + return result; } TNamedFunction TSqlMatchRecognizeClause::ParseOneMeasure(const TRule_row_pattern_measure_definition& node) { - TColumnRefScope scope(Ctx, EColumnRefState::MatchRecognize); - const auto& expr = TSqlExpression(Ctx, Mode).Build(node.GetRule_expr1()); - const auto& name = Id(node.GetRule_an_id3(), *this); - //Each measure must be a lambda, that accepts 2 args: + TColumnRefScope scope(Ctx, EColumnRefState::MatchRecognizeMeasures); + auto callable = TSqlExpression(Ctx, Mode).Build(node.GetRule_expr1()); + auto measureName = Id(node.GetRule_an_id3(), *this); + // Each measure must be a lambda, that accepts 2 args: // - List<InputTableColumns + _yql_Classifier, _yql_MatchNumber> // - Struct that maps row pattern variables to ranges in the queue - return {expr, name}; + return {std::move(callable), std::move(measureName)}; } -TVector<TNamedFunction> TSqlMatchRecognizeClause::ParseMeasures(const TRule_row_pattern_measure_list& node) { - TVector<TNamedFunction> result{ ParseOneMeasure(node.GetRule_row_pattern_measure_definition1()) }; - for (const auto& m: node.GetBlock2()) { +TVector<TNamedFunction> TSqlMatchRecognizeClause::ParseMeasures(const TRule_row_pattern_measure_list* node) { + if (!node) { + return {}; + } + TVector<TNamedFunction> result{ParseOneMeasure(node->GetRule_row_pattern_measure_definition1())}; + for (const auto& m: node->GetBlock2()) { result.push_back(ParseOneMeasure(m.GetRule_row_pattern_measure_definition2())); } return result; } -std::pair<TPosition, NYql::NMatchRecognize::ERowsPerMatch> TSqlMatchRecognizeClause::ParseRowsPerMatch(const TRule_row_pattern_rows_per_match& rowsPerMatchClause) { - - switch(rowsPerMatchClause.GetAltCase()) { - case TRule_row_pattern_rows_per_match::kAltRowPatternRowsPerMatch1: - return std::pair { - TokenPosition(rowsPerMatchClause.GetAlt_row_pattern_rows_per_match1().GetToken1()), - NYql::NMatchRecognize::ERowsPerMatch::OneRow - }; - case TRule_row_pattern_rows_per_match::kAltRowPatternRowsPerMatch2: - return std::pair { - TokenPosition(rowsPerMatchClause.GetAlt_row_pattern_rows_per_match2().GetToken1()), - NYql::NMatchRecognize::ERowsPerMatch::AllRows - }; +TNodePtr TSqlMatchRecognizeClause::ParseRowsPerMatch(TPosition pos, const TRule_row_pattern_rows_per_match* node) { + const auto result = [&]() -> NYql::NMatchRecognize::ERowsPerMatch { + if (!node) { + return NYql::NMatchRecognize::ERowsPerMatch::OneRow; + } + switch (node->GetAltCase()) { + case TRule_row_pattern_rows_per_match::kAltRowPatternRowsPerMatch1: { + const auto& rowsPerMatch = node->GetAlt_row_pattern_rows_per_match1(); + pos = GetPos(rowsPerMatch.GetToken1()); + return NYql::NMatchRecognize::ERowsPerMatch::OneRow; + } + case TRule_row_pattern_rows_per_match::kAltRowPatternRowsPerMatch2: { + const auto& rowsPerMatch = node->GetAlt_row_pattern_rows_per_match2(); + pos = GetPos(rowsPerMatch.GetToken1()); + return NYql::NMatchRecognize::ERowsPerMatch::AllRows; + } case TRule_row_pattern_rows_per_match::ALT_NOT_SET: Y_ABORT("You should change implementation according to grammar changes"); - } + } + }(); + return BuildQuotedAtom(pos, "RowsPerMatch_" + ToString(result)); } -std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo> TSqlMatchRecognizeClause::ParseAfterMatchSkipTo(const TRule_row_pattern_skip_to& skipToClause) { - switch (skipToClause.GetAltCase()) { - case TRule_row_pattern_skip_to::kAltRowPatternSkipTo1: - return std::pair{ - TokenPosition(skipToClause.GetAlt_row_pattern_skip_to1().GetToken1()), - NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::NextRow, ""} - }; - case TRule_row_pattern_skip_to::kAltRowPatternSkipTo2: - return std::pair{ - TokenPosition(skipToClause.GetAlt_row_pattern_skip_to2().GetToken1()), - NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::PastLastRow, ""} - }; - case TRule_row_pattern_skip_to::kAltRowPatternSkipTo3: - return std::pair{ - TokenPosition(skipToClause.GetAlt_row_pattern_skip_to3().GetToken1()), - NYql::NMatchRecognize::TAfterMatchSkipTo{ - NYql::NMatchRecognize::EAfterMatchSkipTo::ToFirst, - skipToClause.GetAlt_row_pattern_skip_to3().GetRule_row_pattern_skip_to_variable_name4().GetRule_row_pattern_variable_name1().GetRule_identifier1().GetToken1().GetValue() - } - }; - case TRule_row_pattern_skip_to::kAltRowPatternSkipTo4: - return std::pair{ - TokenPosition(skipToClause.GetAlt_row_pattern_skip_to4().GetToken1()), - NYql::NMatchRecognize::TAfterMatchSkipTo{ - NYql::NMatchRecognize::EAfterMatchSkipTo::ToLast, - skipToClause.GetAlt_row_pattern_skip_to4().GetRule_row_pattern_skip_to_variable_name4().GetRule_row_pattern_variable_name1().GetRule_identifier1().GetToken1().GetValue() - } - }; - case TRule_row_pattern_skip_to::kAltRowPatternSkipTo5: - return std::pair{ - TokenPosition(skipToClause.GetAlt_row_pattern_skip_to5().GetToken1()), - NYql::NMatchRecognize::TAfterMatchSkipTo{ - NYql::NMatchRecognize::EAfterMatchSkipTo::To, - skipToClause.GetAlt_row_pattern_skip_to5().GetRule_row_pattern_skip_to_variable_name3().GetRule_row_pattern_variable_name1().GetRule_identifier1().GetToken1().GetValue() - } - }; +TNodePtr TSqlMatchRecognizeClause::ParseAfterMatchSkipTo(TPosition pos, const TRule_row_pattern_skip_to* node) { + auto skipToPos = pos; + auto varPos = pos; + const auto result = [&]() -> TMaybe<NYql::NMatchRecognize::TAfterMatchSkipTo> { + if (!node) { + return NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::PastLastRow, ""}; + } + switch (node->GetAltCase()) { + case TRule_row_pattern_skip_to::kAltRowPatternSkipTo1: { + const auto& skipTo = node->GetAlt_row_pattern_skip_to1(); + skipToPos = GetPos(skipTo.GetToken1()); + return NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::NextRow, ""}; + } + case TRule_row_pattern_skip_to::kAltRowPatternSkipTo2: { + const auto& skipTo = node->GetAlt_row_pattern_skip_to2(); + skipToPos = GetPos(skipTo.GetToken1()); + return NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::PastLastRow, ""}; + } + case TRule_row_pattern_skip_to::kAltRowPatternSkipTo3: { + const auto& skipTo = node->GetAlt_row_pattern_skip_to3(); + skipToPos = GetPos(skipTo.GetToken1()); + const auto& identifier = skipTo.GetRule_row_pattern_skip_to_variable_name4().GetRule_row_pattern_variable_name1().GetRule_identifier1(); + auto var = identifier.GetToken1().GetValue(); + varPos = GetPos(identifier.GetToken1()); + if (!PatternVarNames.contains(var)) { + Ctx.Error(varPos) << "Unknown pattern variable in AFTER MATCH SKIP TO FIRST"; + return {}; + } + return NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::ToFirst, std::move(var)}; + } + case TRule_row_pattern_skip_to::kAltRowPatternSkipTo4: { + const auto& skipTo = node->GetAlt_row_pattern_skip_to4(); + skipToPos = GetPos(skipTo.GetToken1()); + const auto& identifier = skipTo.GetRule_row_pattern_skip_to_variable_name4().GetRule_row_pattern_variable_name1().GetRule_identifier1(); + auto var = identifier.GetToken1().GetValue(); + varPos = GetPos(identifier.GetToken1()); + if (!PatternVarNames.contains(var)) { + Ctx.Error(varPos) << "Unknown pattern variable in AFTER MATCH SKIP TO LAST"; + return {}; + } + return NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::ToLast, std::move(var)}; + } + case TRule_row_pattern_skip_to::kAltRowPatternSkipTo5: { + const auto& skipTo = node->GetAlt_row_pattern_skip_to5(); + skipToPos = GetPos(skipTo.GetToken1()); + const auto& identifier = skipTo.GetRule_row_pattern_skip_to_variable_name3().GetRule_row_pattern_variable_name1().GetRule_identifier1(); + auto var = identifier.GetToken1().GetValue(); + varPos = GetPos(identifier.GetToken1()); + if (!PatternVarNames.contains(var)) { + Ctx.Error(varPos) << "Unknown pattern variable in AFTER MATCH SKIP TO"; + return {}; + } + return NYql::NMatchRecognize::TAfterMatchSkipTo{NYql::NMatchRecognize::EAfterMatchSkipTo::To, std::move(var)}; + } case TRule_row_pattern_skip_to::ALT_NOT_SET: Y_ABORT("You should change implementation according to grammar changes"); + } + }(); + if (!result) { + return {}; } + return BuildTuple(pos, { + BuildQuotedAtom(skipToPos, "AfterMatchSkip_" + ToString(result->To)), + BuildQuotedAtom(varPos, std::move(result->Var)) + }); } -NYql::NMatchRecognize::TRowPatternTerm TSqlMatchRecognizeClause::ParsePatternTerm(const TRule_row_pattern_term& node, size_t patternNestingLevel, bool outputArg) { - NYql::NMatchRecognize::TRowPatternTerm term; - TPosition pos; - for (const auto& factor: node.GetBlock1()) { - const auto& primaryVar = factor.GetRule_row_pattern_factor1().GetRule_row_pattern_primary1(); - NYql::NMatchRecognize::TRowPatternPrimary primary; - bool output = outputArg; - switch (primaryVar.GetAltCase()) { - case TRule_row_pattern_primary::kAltRowPatternPrimary1: - primary = PatternVar(primaryVar.GetAlt_row_pattern_primary1().GetRule_row_pattern_primary_variable_name1().GetRule_row_pattern_variable_name1(), *this); - break; - case TRule_row_pattern_primary::kAltRowPatternPrimary2: - primary = primaryVar.GetAlt_row_pattern_primary2().GetToken1().GetValue(); - Y_ENSURE("$" == std::get<0>(primary)); - break; - case TRule_row_pattern_primary::kAltRowPatternPrimary3: - primary = primaryVar.GetAlt_row_pattern_primary3().GetToken1().GetValue(); - Y_ENSURE("^" == std::get<0>(primary)); - break; - case TRule_row_pattern_primary::kAltRowPatternPrimary4: { - if (patternNestingLevel <= NYql::NMatchRecognize::MaxPatternNesting) { - primary = ParsePattern(primaryVar.GetAlt_row_pattern_primary4().GetBlock2().GetRule_row_pattern1(), patternNestingLevel + 1, output); - } else { - Ctx.Error(TokenPosition(primaryVar.GetAlt_row_pattern_primary4().GetToken1())) - << "To big nesting level in the pattern"; - return NYql::NMatchRecognize::TRowPatternTerm{}; - } - break; +TNodePtr TSqlMatchRecognizeClause::BuildPatternFactor(TPosition pos, TNodePtr primary, std::tuple<ui64, ui64, bool, bool, bool> quantifier) { + return std::apply([&](const auto& ...args) { + return BuildTuple(pos, {std::move(primary), BuildQuotedAtom(pos, ToString(args))...}); + }, quantifier); +} + +TNodePtr TSqlMatchRecognizeClause::ParsePatternFactor(TPosition pos, const TRule_row_pattern_factor& node, size_t nestingLevel, bool output) { + if (nestingLevel > MaxPatternNesting) { + Ctx.Error(pos) << "To big nesting level in the pattern"; + return {}; + } + auto primary = [&]() -> TNodePtr { + const auto& primaryAlt = node.GetRule_row_pattern_primary1(); + switch (primaryAlt.GetAltCase()) { + case TRule_row_pattern_primary::kAltRowPatternPrimary1: { + const auto& primary = primaryAlt.GetAlt_row_pattern_primary1(); + const auto& identifier = primary.GetRule_row_pattern_primary_variable_name1().GetRule_row_pattern_variable_name1().GetRule_identifier1(); + const auto varName = Id(identifier, *this); + const auto var = BuildQuotedAtom(GetPos(identifier.GetToken1()), varName); + if (PatternVarNames.insert(varName).second) { + PatternVars->Add(var); } - case TRule_row_pattern_primary::kAltRowPatternPrimary5: - output = false; - primary = ParsePattern(primaryVar.GetAlt_row_pattern_primary5().GetRule_row_pattern3(), patternNestingLevel + 1, output); - break; - case TRule_row_pattern_primary::kAltRowPatternPrimary6: { - std::vector<NYql::NMatchRecognize::TRowPatternPrimary> items{ParsePattern( - primaryVar.GetAlt_row_pattern_primary6().GetRule_row_pattern_permute1().GetRule_row_pattern3(), patternNestingLevel + 1, output) - }; - for (const auto& p: primaryVar.GetAlt_row_pattern_primary6().GetRule_row_pattern_permute1().GetBlock4()) { - items.push_back(ParsePattern(p.GetRule_row_pattern2(), patternNestingLevel + 1, output)); - } - //Permutations now is a syntactic sugar and converted to all possible alternatives - if (items.size() > NYql::NMatchRecognize::MaxPermutedItems) { - Ctx.Error(TokenPosition(primaryVar.GetAlt_row_pattern_primary4().GetToken1())) - << "Too many items in permute"; - return NYql::NMatchRecognize::TRowPatternTerm{}; - } - std::vector<size_t> indexes(items.size()); - std::generate(begin(indexes), end(indexes), [n = 0] () mutable { return n++; }); - NYql::NMatchRecognize::TRowPattern permuted; - do { - NYql::NMatchRecognize::TRowPatternTerm term; - term.reserve(indexes.size()); - for (size_t i = 0; i != indexes.size(); ++i) { - term.push_back({items[indexes[i]], 1, 1, true, false, false}); - } - permuted.push_back(std::move(term)); - } while (std::next_permutation(indexes.begin(), indexes.end())); - primary = permuted; - break; + return var; + } + case TRule_row_pattern_primary::kAltRowPatternPrimary2: { + const auto& primary = primaryAlt.GetAlt_row_pattern_primary2(); + const auto& token = primary.GetToken1(); + const auto varName = token.GetValue(); + const auto var = BuildQuotedAtom(GetPos(token), varName); + if (PatternVarNames.insert(varName).second) { + PatternVars->Add(var); } - case TRule_row_pattern_primary::ALT_NOT_SET: - Y_ABORT("You should change implementation according to grammar changes"); + return var; } - uint64_t quantityMin = 1; - uint64_t quantityMax = 1; - constexpr uint64_t infinity = std::numeric_limits<uint64_t>::max(); - bool greedy = true; - if (factor.GetRule_row_pattern_factor1().HasBlock2()) { - const auto& quantifier = factor.GetRule_row_pattern_factor1().GetBlock2().GetRule_row_pattern_quantifier1(); - switch(quantifier.GetAltCase()){ - case TRule_row_pattern_quantifier::kAltRowPatternQuantifier1: //* - quantityMin = 0; - quantityMax = infinity; - greedy = !quantifier.GetAlt_row_pattern_quantifier1().HasBlock2(); - break; - case TRule_row_pattern_quantifier::kAltRowPatternQuantifier2: //+ - quantityMax = infinity; - greedy = !quantifier.GetAlt_row_pattern_quantifier2().HasBlock2(); - break; - case TRule_row_pattern_quantifier::kAltRowPatternQuantifier3: //? - quantityMin = 0; - greedy = !quantifier.GetAlt_row_pattern_quantifier3().HasBlock2(); - break; - case TRule_row_pattern_quantifier::kAltRowPatternQuantifier4: //{ 2?, 4?} - if (quantifier.GetAlt_row_pattern_quantifier4().HasBlock2()) { - quantityMin = FromString(quantifier.GetAlt_row_pattern_quantifier4().GetBlock2().GetRule_integer1().GetToken1().GetValue()); - } - else { - quantityMin = 0;; - } - if (quantifier.GetAlt_row_pattern_quantifier4().HasBlock4()) { - quantityMax = FromString(quantifier.GetAlt_row_pattern_quantifier4().GetBlock4().GetRule_integer1().GetToken1().GetValue()); - } - else { - quantityMax = infinity; - } - greedy = !quantifier.GetAlt_row_pattern_quantifier4().HasBlock6(); - - break; - case TRule_row_pattern_quantifier::kAltRowPatternQuantifier5: - quantityMin = quantityMax = FromString(quantifier.GetAlt_row_pattern_quantifier5().GetRule_integer2().GetToken1().GetValue()); - break; - case TRule_row_pattern_quantifier::ALT_NOT_SET: - Y_ABORT("You should change implementation according to grammar changes"); + case TRule_row_pattern_primary::kAltRowPatternPrimary3: { + const auto& primary = primaryAlt.GetAlt_row_pattern_primary3(); + const auto& token = primary.GetToken1(); + const auto varName = token.GetValue(); + const auto var = BuildQuotedAtom(GetPos(token), varName); + if (PatternVarNames.insert(varName).second) { + PatternVars->Add(var); } + return var; + } + case TRule_row_pattern_primary::kAltRowPatternPrimary4: { + const auto& primary = primaryAlt.GetAlt_row_pattern_primary4(); + return ParsePattern(pos, primary.GetBlock2().GetRule_row_pattern1(), nestingLevel + 1, output); + } + case TRule_row_pattern_primary::kAltRowPatternPrimary5: { + const auto& primary = primaryAlt.GetAlt_row_pattern_primary5(); + output = false; + return ParsePattern(pos, primary.GetRule_row_pattern3(), nestingLevel + 1, output); + } + case TRule_row_pattern_primary::kAltRowPatternPrimary6: { + const auto& primary = primaryAlt.GetAlt_row_pattern_primary6(); + std::vector<TNodePtr> items{ + ParsePattern(pos, primary.GetRule_row_pattern_permute1().GetRule_row_pattern3(), nestingLevel + 1, output) + }; + for (const auto& p: primary.GetRule_row_pattern_permute1().GetBlock4()) { + items.push_back(ParsePattern(pos, p.GetRule_row_pattern2(), nestingLevel + 1, output)); + } + if (items.size() > MaxPermutedItems) { + Ctx.Error(GetPos(primary.GetRule_row_pattern_permute1().GetToken1())) << "Too many items in permute"; + return {}; + } + std::vector<size_t> indexes(items.size()); + Iota(indexes.begin(), indexes.end(), 0); + std::vector<TNodePtr> result; + do { + std::vector<TNodePtr> term; + term.reserve(items.size()); + for (auto index : indexes) { + term.push_back(BuildPatternFactor(pos, items[index], std::tuple{1, 1, true, output, false})); + } + result.push_back(BuildPatternTerm(pos, std::move(term))); + } while (std::next_permutation(indexes.begin(), indexes.end())); + return BuildPattern(pos, std::move(result)); + } + case TRule_row_pattern_primary::ALT_NOT_SET: + Y_ABORT("You should change implementation according to grammar changes"); } - term.push_back(NYql::NMatchRecognize::TRowPatternFactor{std::move(primary), quantityMin, quantityMax, greedy, output, false}); + }(); + if (!primary) { + return {}; } - return term; + + const auto quantifier = [&]() { + if (!node.HasBlock2()) { + const auto quantity = static_cast<ui64>(1); + return std::tuple{quantity, quantity, true, output, false}; + } + const auto& quantifierAlt = node.GetBlock2().GetRule_row_pattern_quantifier1(); + switch (quantifierAlt.GetAltCase()) { + case TRule_row_pattern_quantifier::kAltRowPatternQuantifier1: { // * + const auto& quantifier = quantifierAlt.GetAlt_row_pattern_quantifier1(); + pos = GetPos(quantifier.GetToken1()); + return std::tuple{static_cast<ui64>(0), static_cast<ui64>(Max()), !quantifier.HasBlock2(), output, false}; + } + case TRule_row_pattern_quantifier::kAltRowPatternQuantifier2: { // + + const auto& quantifier = quantifierAlt.GetAlt_row_pattern_quantifier2(); + pos = GetPos(quantifier.GetToken1()); + return std::tuple{static_cast<ui64>(1), static_cast<ui64>(Max()), !quantifier.HasBlock2(), output, false}; + } + case TRule_row_pattern_quantifier::kAltRowPatternQuantifier3: { // ? + const auto& quantifier = quantifierAlt.GetAlt_row_pattern_quantifier3(); + pos = GetPos(quantifier.GetToken1()); + return std::tuple{static_cast<ui64>(0), static_cast<ui64>(1), !quantifier.HasBlock2(), output, false}; + } + case TRule_row_pattern_quantifier::kAltRowPatternQuantifier4: { // {n?, m?} + const auto& quantifier = quantifierAlt.GetAlt_row_pattern_quantifier4(); + pos = GetPos(quantifier.GetToken1()); + return std::tuple{ + quantifier.HasBlock2() + ? FromString(quantifier.GetBlock2().GetRule_integer1().GetToken1().GetValue()) + : static_cast<ui64>(0), + quantifier.HasBlock4() + ? FromString(quantifier.GetBlock4().GetRule_integer1().GetToken1().GetValue()) + : static_cast<ui64>(Max()), + !quantifier.HasBlock6(), + output, + false + }; + } + case TRule_row_pattern_quantifier::kAltRowPatternQuantifier5: { // {n} + const auto quantifier = quantifierAlt.GetAlt_row_pattern_quantifier5(); + pos = GetPos(quantifier.GetToken1()); + const auto quantity = static_cast<ui64>(FromString(quantifier.GetRule_integer2().GetToken1().GetValue())); + return std::tuple{quantity, quantity, true, output, false}; + } + case TRule_row_pattern_quantifier::ALT_NOT_SET: + Y_ABORT("You should change implementation according to grammar changes"); + } + }(); + return BuildPatternFactor(pos, std::move(primary), std::move(quantifier)); } -NYql::NMatchRecognize::TRowPattern TSqlMatchRecognizeClause::ParsePattern(const TRule_row_pattern& node, size_t patternNestingLevel, bool output){ - TVector<NYql::NMatchRecognize::TRowPatternTerm> result; - result.push_back(ParsePatternTerm(node.GetRule_row_pattern_term1(), patternNestingLevel, output)); - for (const auto& term: node.GetBlock2()) - result.push_back(ParsePatternTerm(term.GetRule_row_pattern_term2(), patternNestingLevel, output)); +TNodePtr TSqlMatchRecognizeClause::BuildPatternTerm(TPosition pos, std::vector<TNodePtr> term) { + auto result = BuildList(pos); + for (auto& factor : term) { + if (!factor) { + return {}; + } + result->Add(std::move(factor)); + } + return BuildQuote(pos, std::move(result)); +} + +TNodePtr TSqlMatchRecognizeClause::ParsePatternTerm(TPosition pos, const TRule_row_pattern_term& node, size_t nestingLevel, bool output) { + std::vector<TNodePtr> result; + result.reserve(node.GetBlock1().size()); + for (const auto& factor: node.GetBlock1()) { + result.push_back(ParsePatternFactor(pos, factor.GetRule_row_pattern_factor1(), nestingLevel, output)); + } + return BuildPatternTerm(pos, std::move(result)); +} + +TNodePtr TSqlMatchRecognizeClause::BuildPattern(TPosition pos, std::vector<TNodePtr> pattern) { + const auto result = BuildList(pos, {BuildAtom(pos, "MatchRecognizePattern")}); + for (auto& term: pattern) { + if (!term) { + return {}; + } + result->Add(std::move(term)); + } return result; } -TNamedFunction TSqlMatchRecognizeClause::ParseOneDefinition(const TRule_row_pattern_definition& node){ - const auto& varName = PatternVar(node.GetRule_row_pattern_definition_variable_name1().GetRule_row_pattern_variable_name1(), *this); - TColumnRefScope scope(Ctx, EColumnRefState::MatchRecognize, true, varName); - const auto& searchCondition = TSqlExpression(Ctx, Mode).Build(node.GetRule_row_pattern_definition_search_condition3().GetRule_search_condition1().GetRule_expr1()); - return TNamedFunction{searchCondition, varName}; +TNodePtr TSqlMatchRecognizeClause::ParsePattern(TPosition pos, const TRule_row_pattern& node, size_t nestingLevel, bool output) { + std::vector<TNodePtr> result; + result.reserve(1 + node.GetBlock2().size()); + result.push_back(ParsePatternTerm(pos, node.GetRule_row_pattern_term1(), nestingLevel, output)); + for (const auto& term: node.GetBlock2()) { + result.push_back(ParsePatternTerm(pos, term.GetRule_row_pattern_term2(), nestingLevel, output)); + } + return BuildPattern(pos, std::move(result)); +} + +TMaybe<TNodePtr> TSqlMatchRecognizeClause::ParseSubset(TPosition pos, const TRule_row_pattern_subset_clause* node) { + if (!node) { + return TNodePtr{}; + } + pos = GetPos(node->GetToken1()); + // TODO https://st.yandex-team.ru/YQL-16225 + Ctx.Error(pos) << "SUBSET is not implemented yet"; + return {}; +} + +TNamedFunction TSqlMatchRecognizeClause::ParseOneDefinition(const TRule_row_pattern_definition& node) { + const auto& identifier = node.GetRule_row_pattern_definition_variable_name1().GetRule_row_pattern_variable_name1().GetRule_identifier1(); + auto defineName = Id(identifier, *this); + TColumnRefScope scope(Ctx, EColumnRefState::MatchRecognizeDefine, true, defineName); + const auto& searchCondition = node.GetRule_row_pattern_definition_search_condition3().GetRule_search_condition1().GetRule_expr1(); + auto callable = TSqlExpression(Ctx, Mode).Build(searchCondition); + // Each define must be a predicate lambda, that accepts 3 args: + // - List<input table rows> + // - A struct that maps row pattern variables to ranges in the queue + // - An index of the current row + return {std::move(callable), std::move(defineName)}; } TVector<TNamedFunction> TSqlMatchRecognizeClause::ParseDefinitions(const TRule_row_pattern_definition_list& node) { - TVector<TNamedFunction> result { ParseOneDefinition(node.GetRule_row_pattern_definition1())}; + TVector<TNamedFunction> result{ParseOneDefinition(node.GetRule_row_pattern_definition1())}; for (const auto& d: node.GetBlock2()) { - //Each define must be a predicate lambda, that accepts 3 args: - // - List<input table rows> - // - A struct that maps row pattern variables to ranges in the queue - // - An index of the current row result.push_back(ParseOneDefinition(d.GetRule_row_pattern_definition2())); } return result; } -} //namespace NSQLTranslationV1 +} // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/sql_match_recognize.h b/yql/essentials/sql/v1/sql_match_recognize.h index 219baeaa093..928ed35b5db 100644 --- a/yql/essentials/sql/v1/sql_match_recognize.h +++ b/yql/essentials/sql/v1/sql_match_recognize.h @@ -1,28 +1,38 @@ #pragma once -#include "sql_translation.h" #include "match_recognize.h" +#include "node.h" +#include "sql_translation.h" namespace NSQLTranslationV1 { -using namespace NSQLv1Generated; - -class TSqlMatchRecognizeClause: public TSqlTranslation { +class TSqlMatchRecognizeClause final : public TSqlTranslation { public: - TSqlMatchRecognizeClause(TContext& ctx, NSQLTranslation::ESqlMode mode) - : TSqlTranslation(ctx, mode) - {} + TSqlMatchRecognizeClause(TContext& ctx, NSQLTranslation::ESqlMode mode); TMatchRecognizeBuilderPtr CreateBuilder(const TRule_row_pattern_recognition_clause& node); + static constexpr size_t MaxPatternNesting = 20; + static constexpr size_t MaxPermutedItems = 6; + private: - TVector<TNamedFunction> ParsePartitionBy(const TRule_window_partition_clause& partitionClause); + std::tuple<TNodePtr, TNodePtr> ParsePartitionBy(TPosition pos, const TRule_window_partition_clause* node); + TMaybe<TVector<TSortSpecificationPtr>> ParseOrderBy(const TRule_order_by_clause* node); TNamedFunction ParseOneMeasure(const TRule_row_pattern_measure_definition& node); - TVector<TNamedFunction> ParseMeasures(const TRule_row_pattern_measure_list& node); - std::pair<TPosition, NYql::NMatchRecognize::ERowsPerMatch> ParseRowsPerMatch(const TRule_row_pattern_rows_per_match& rowsPerMatchClause); - std::pair<TPosition, NYql::NMatchRecognize::TAfterMatchSkipTo> ParseAfterMatchSkipTo(const TRule_row_pattern_skip_to& skipToClause); - NYql::NMatchRecognize::TRowPatternTerm ParsePatternTerm(const TRule_row_pattern_term& node, size_t patternNestingLevel, bool output); - NYql::NMatchRecognize::TRowPattern ParsePattern(const TRule_row_pattern& node, size_t patternNestingLevel = 1, bool output = true); + TVector<TNamedFunction> ParseMeasures(const TRule_row_pattern_measure_list* node); + TNodePtr ParseRowsPerMatch(TPosition pos, const TRule_row_pattern_rows_per_match* node); + TNodePtr ParseAfterMatchSkipTo(TPosition pos, const TRule_row_pattern_skip_to* node); + TNodePtr BuildPatternFactor(TPosition pos, TNodePtr primary, std::tuple<ui64, ui64, bool, bool, bool> quantifier); + TNodePtr ParsePatternFactor(TPosition pos, const TRule_row_pattern_factor& node, size_t nestingLevel, bool output); + TNodePtr BuildPatternTerm(TPosition pos, std::vector<TNodePtr> term); + TNodePtr ParsePatternTerm(TPosition pos, const TRule_row_pattern_term& node, size_t nestingLevel, bool output); + TNodePtr BuildPattern(TPosition pos, std::vector<TNodePtr> pattern); + TNodePtr ParsePattern(TPosition pos, const TRule_row_pattern& node, size_t nestingLevel, bool output); + TMaybe<TNodePtr> ParseSubset(TPosition pos, const TRule_row_pattern_subset_clause* node); TNamedFunction ParseOneDefinition(const TRule_row_pattern_definition& node); TVector<TNamedFunction> ParseDefinitions(const TRule_row_pattern_definition_list& node); + +private: + THashSet<TString> PatternVarNames; + TNodePtr PatternVars; }; } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/sql_match_recognize_ut.cpp b/yql/essentials/sql/v1/sql_match_recognize_ut.cpp index f591ef06474..f599dc6e991 100644 --- a/yql/essentials/sql/v1/sql_match_recognize_ut.cpp +++ b/yql/essentials/sql/v1/sql_match_recognize_ut.cpp @@ -41,7 +41,7 @@ bool IsLambda(const NYql::TAstNode* node, ui32 numberOfArgs) { return false; } if (!node->GetChild(0)->IsAtom() || node->GetChild(0)->GetContent() != "lambda") { - return false; + return false; } return IsQuotedListOfSize(node->GetChild(1), numberOfArgs); } @@ -71,7 +71,7 @@ FROM Input MATCH_RECOGNIZE( auto matchRecognizeAndSample = R"( USE plato; SELECT * -FROM Input MATCH_RECOGNIZE( +FROM Input MATCH_RECOGNIZE( PATTERN ( A ) DEFINE A as A ) TABLESAMPLE BERNOULLI(1.0) @@ -148,13 +148,13 @@ FROM Input MATCH_RECOGNIZE( auto r = MatchRecognizeSqlToYql(stmt); UNIT_ASSERT(r.IsOk()); const auto measures = FindMatchRecognizeParam(r.Root, "measures"); - UNIT_ASSERT_VALUES_EQUAL(6, measures->GetChildrenCount()); + UNIT_ASSERT_VALUES_EQUAL(7, measures->GetChildrenCount()); const auto columnNames = measures->GetChild(3); UNIT_ASSERT(IsQuotedListOfSize(columnNames, 2)); UNIT_ASSERT_VALUES_EQUAL("T", columnNames->GetChild(1)->GetChild(0)->GetChild(1)->GetContent()); UNIT_ASSERT_VALUES_EQUAL("Key", columnNames->GetChild(1)->GetChild(1)->GetChild(1)->GetContent()); - UNIT_ASSERT(IsLambda(measures->GetChild(4), 2)); - UNIT_ASSERT(IsLambda(measures->GetChild(5), 2)); + UNIT_ASSERT(IsQuotedListOfSize(measures->GetChild(4), 2)); + UNIT_ASSERT(IsQuotedListOfSize(measures->GetChild(5), 2)); } Y_UNIT_TEST(RowsPerMatch) { { @@ -326,7 +326,7 @@ USE plato; SELECT * FROM Input MATCH_RECOGNIZE( INITIAL - PATTERN (A+ B* C?) + PATTERN (A+ B* C?) DEFINE A as A ) )"; @@ -340,7 +340,7 @@ USE plato; SELECT * FROM Input MATCH_RECOGNIZE( SEEK - PATTERN (A+ B* C?) + PATTERN (A+ B* C?) DEFINE A as A ) )"; @@ -353,7 +353,7 @@ FROM Input MATCH_RECOGNIZE( USE plato; SELECT * FROM Input MATCH_RECOGNIZE( - PATTERN (A+ B* C?) + PATTERN (A+ B* C?) DEFINE A as A ) )"; @@ -427,7 +427,7 @@ PATTERN ( } Y_UNIT_TEST(PatternLimitedNesting) { - const size_t MaxNesting = 20; + constexpr size_t MaxNesting = 20; for (size_t extraNesting = 0; extraNesting <= 1; ++extraNesting) { std::string pattern; for (size_t i = 0; i != MaxNesting + extraNesting; ++i) @@ -469,7 +469,7 @@ FROM Input MATCH_RECOGNIZE( }; auto getTheFactor = [](const NYql::TAstNode* root) { const auto& patternCallable = FindMatchRecognizeParam(root, "pattern"); - const auto& factor = patternCallable->GetChild(1)->GetChild(1)->GetChild(0)->GetChild(1); + const auto& factor = patternCallable->GetChild(1)->GetChild(1)->GetChild(0)->GetChild(1); return NYql::NMatchRecognize::TRowPatternFactor{ TString(), //primary var or subexpression, not used in this test FromString<uint64_t>(factor->GetChild(1)->GetChild(1)->GetContent()), //QuantityMin @@ -651,7 +651,7 @@ FROM Input MATCH_RECOGNIZE( } Y_UNIT_TEST(PermuteTooMuch) { - for (size_t n = 1; n <= NYql::NMatchRecognize::MaxPermutedItems + 1; ++n) { + for (size_t n = 1; n <= 6 + 1; ++n) { std::vector<std::string> vars(n); std::generate(begin(vars), end(vars), [n = 0] () mutable { return "A" + std::to_string(n++);}); const auto stmt = TString(R"( @@ -671,7 +671,7 @@ FROM Input MATCH_RECOGNIZE( )" ); const auto &r = MatchRecognizeSqlToYql(stmt); - if (n <= NYql::NMatchRecognize::MaxPermutedItems) { + if (n <= 6) { UNIT_ASSERT(r.IsOk()); } else { UNIT_ASSERT(!r.IsOk()); diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp index 6bdd2c7e441..bd385c8feb5 100644 --- a/yql/essentials/sql/v1/sql_ut.cpp +++ b/yql/essentials/sql/v1/sql_ut.cpp @@ -8162,3 +8162,42 @@ Y_UNIT_TEST_SUITE(QuerySplit) { };)"); } } + +Y_UNIT_TEST_SUITE(MatchRecognizeMeasuresAggregation) { + Y_UNIT_TEST(InsideSelect) { + ExpectFailWithError(R"sql( + SELECT FIRST(0), LAST(1); + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(OutsideSelect) { + ExpectFailWithError(R"sql( + $lambda = ($x) -> (FIRST($x) + LAST($x)); + SELECT $lambda(x) FROM plato.Input; + )sql", + "<main>:2:32: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:44: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(AsAggregateFunction) { + ExpectFailWithError(R"sql( + SELECT FIRST(x), LAST(x) FROM plato.Input; + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(AsWindowFunction) { + ExpectFailWithError(R"sql( + SELECT FIRST(x) OVER(), LAST(x) OVER() FROM plato.Input; + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:37: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } +} diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp index 4b1233bfee2..9463408886e 100644 --- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp +++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp @@ -8121,3 +8121,42 @@ $__ydb_transfer_lambda = ($x) -> { } } + +Y_UNIT_TEST_SUITE(MatchRecognizeMeasuresAggregation) { + Y_UNIT_TEST(InsideSelect) { + ExpectFailWithError(R"sql( + SELECT FIRST(0), LAST(1); + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(OutsideSelect) { + ExpectFailWithError(R"sql( + $lambda = ($x) -> (FIRST($x) + LAST($x)); + SELECT $lambda(x) FROM plato.Input; + )sql", + "<main>:2:32: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:44: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(AsAggregateFunction) { + ExpectFailWithError(R"sql( + SELECT FIRST(x), LAST(x) FROM plato.Input; + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:30: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } + + Y_UNIT_TEST(AsWindowFunction) { + ExpectFailWithError(R"sql( + SELECT FIRST(x) OVER(), LAST(x) OVER() FROM plato.Input; + )sql", + "<main>:2:20: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + "<main>:2:37: Error: Cannot use FIRST and LAST outside the MATCH_RECOGNIZE context\n" + ); + } +} |