diff options
Diffstat (limited to 'yql/essentials/sql')
17 files changed, 312 insertions, 75 deletions
diff --git a/yql/essentials/sql/settings/translation_settings.cpp b/yql/essentials/sql/settings/translation_settings.cpp index 81cf2785150..02fd18501b2 100644 --- a/yql/essentials/sql/settings/translation_settings.cpp +++ b/yql/essentials/sql/settings/translation_settings.cpp @@ -6,6 +6,7 @@ #include <library/cpp/deprecated/split/split_iterator.h> +#include <util/string/cast.h> #include <util/string/split.h> #include <util/string/join.h> #include <util/system/env.h> @@ -33,7 +34,7 @@ public: }; [[noreturn]] TString ThrowBad(TStringBuf flag, const TVector<TString>& args) { - YQL_ENSURE(false, "Bad " << flag << "args [" << JoinSeq(", ", args) << "]"); + YQL_ENSURE(false, "Bad " << flag << " args [" << JoinSeq(", ", args) << "]"); } } // namespace @@ -188,6 +189,21 @@ void ParseTranslationSettings(const TExtendedSqlFlags& flags, TTranslationSettin } }, }, + { + "MaxParseTreeDepth", + [](const TVector<TString>& args, TTranslationSettings& s) { + if (args.size() != 1) { + ThrowBad("MaxParseTreeDepth", args); + } + + size_t value = 0; + if (!TryFromString(args[0], value)) { + ThrowBad("MaxParseTreeDepth", args); + } + + s.MaxParseTreeDepth = value; + }, + }, }; for (const auto& [flag, args] : flags) { diff --git a/yql/essentials/sql/settings/translation_settings.h b/yql/essentials/sql/settings/translation_settings.h index 9252d84caf6..a5598aa6eb8 100644 --- a/yql/essentials/sql/settings/translation_settings.h +++ b/yql/essentials/sql/settings/translation_settings.h @@ -21,7 +21,9 @@ class IAutoParamBuilderFactory; } // namespace NYql namespace NSQLTranslation { + constexpr const size_t SQL_MAX_PARSER_ERRORS = 100; +constexpr const size_t SQL_MAX_PARSE_TREE_DEPTH = 4096; enum class ESqlMode { QUERY = 0, @@ -121,6 +123,7 @@ struct TTranslationSettings { bool PGDisable; bool WarnOnV0; bool TestAntlr4; // TODO(YQL-19017): remove. + TMaybe<size_t> MaxParseTreeDepth; ISqlFeaturePolicy::TPtr V0WarnAsError; ISqlFeaturePolicy::TPtr DqDefaultAuto; ISqlFeaturePolicy::TPtr BlockDefaultAuto; diff --git a/yql/essentials/sql/v1/format/check/check_format.cpp b/yql/essentials/sql/v1/format/check/check_format.cpp index 41ca6dfedf2..a1e891df807 100644 --- a/yql/essentials/sql/v1/format/check/check_format.cpp +++ b/yql/essentials/sql/v1/format/check/check_format.cpp @@ -133,8 +133,14 @@ TMaybe<TString> CheckedFormat( }; NSQLTranslationV1::TParsers parsers = { - .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory(), - .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory(), + .Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), + .Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth), }; auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings); diff --git a/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp b/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp index a0d7b448df0..bb8ad16d72d 100644 --- a/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp +++ b/yql/essentials/sql/v1/ide/completion/check/check_complete.cpp @@ -12,6 +12,8 @@ #include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h> #include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h> +#include <contrib/libs/antlr4_cpp_runtime/src/Exceptions.h> + #include <util/charset/utf8.h> #include <util/random/random.h> @@ -93,15 +95,21 @@ bool CheckComplete(TStringBuf query, TYqlContext ctx) { bool CheckComplete(TStringBuf query, NYql::TExprNode::TPtr root, NYql::TExprContext& ctx, NYql::TIssues& issues) try { return CheckComplete(query, MakeYqlAnalysis()->Analyze(root, ctx)); +} catch (const antlr4::ParseCancellationException& e) { + Y_UNUSED(e); + return true; } catch (...) { - issues.AddIssue(FormatCurrentException()); + issues.AddIssue(CurrentExceptionMessage()); return false; } bool CheckComplete(TStringBuf query, NYql::TAstNode& root, NYql::TIssues& issues) try { return CheckComplete(query, MakeYqlAnalysis()->Analyze(root, issues).GetOrElse({})); +} catch (const antlr4::ParseCancellationException& e) { + Y_UNUSED(e); + return true; } catch (...) { - issues.AddIssue(FormatCurrentException()); + issues.AddIssue(CurrentExceptionMessage()); return false; } diff --git a/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp b/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp index 092f7803a5e..7f2e1511e3d 100644 --- a/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/ide/completion/sql_complete_ut.cpp @@ -2401,4 +2401,23 @@ Y_UNIT_TEST(CachedSchema) { } } +Y_UNIT_TEST(NoStackOverflowOnDeeplyNestedSubquery) { + constexpr size_t Depth = 4 * 1024; + + auto engine = MakeSqlCompletionEngineUT(); + + TStringBuilder query; + query << '#'; + for (size_t i = 0; i < Depth; ++i) { + query << "SELECT * FROM ("; + } + query << "SELECT 1"; + for (size_t i = 0; i < Depth; ++i) { + query << ")"; + } + + UNIT_ASSERT_EXCEPTION_CONTAINS( + Complete(engine, query), std::exception, "Maximum parse tree depth exceeded"); +} + } // Y_UNIT_TEST_SUITE(SqlCompleteTests) diff --git a/yql/essentials/sql/v1/ide/pure_ast/parser.cpp b/yql/essentials/sql/v1/ide/pure_ast/parser.cpp index 4b9fd1e8d59..0b0e6d0805a 100644 --- a/yql/essentials/sql/v1/ide/pure_ast/parser.cpp +++ b/yql/essentials/sql/v1/ide/pure_ast/parser.cpp @@ -2,11 +2,13 @@ #include "parse_tree.h" +#include <yql/essentials/parser/common/antlr4/depth_limiting_listener.h> #include <yql/essentials/parser/antlr_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> #include <yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> #include <util/system/yassert.h> #include <util/charset/utf8.h> +#include <util/generic/maybe.h> #include <util/string/builder.h> namespace NSQLPureAST { @@ -22,6 +24,8 @@ public: template <bool IsAnsiLexer> class TParser: public IParser { + static constexpr size_t MaxParseTreeDepth = 4096; + public: using TLexer = std::conditional_t< IsAnsiLexer, @@ -32,11 +36,13 @@ public: : Chars_() , Lexer_(&Chars_) , Tokens_(&Lexer_) + , DepthLimiter_(/*maxDepth=*/MaxParseTreeDepth) , Parser_(&Tokens_) { Lexer_.removeErrorListeners(); Parser_.removeErrorListeners(); Parser_.setErrorHandler(std::make_shared<TErrorStrategy>()); + Parser_.addParseListener(&DepthLimiter_); } TParseTree Parse(TStringBuf text) override { @@ -61,6 +67,7 @@ private: Chars_.load(text.Data(), text.Size(), /* lenient = */ false); Lexer_.reset(); Tokens_.setTokenSource(&Lexer_); + DepthLimiter_.Reset(); Parser_.reset(); return Parser_.sql_query(); } @@ -89,6 +96,7 @@ private: antlr4::ANTLRInputStream Chars_; TLexer Lexer_; antlr4::CommonTokenStream Tokens_; + NAntlrAST::TDepthLimitingListener DepthLimiter_; SQLv1 Parser_; }; diff --git a/yql/essentials/sql/v1/ide/pure_ast/ya.make b/yql/essentials/sql/v1/ide/pure_ast/ya.make index ed83b11e704..c0eab67f51d 100644 --- a/yql/essentials/sql/v1/ide/pure_ast/ya.make +++ b/yql/essentials/sql/v1/ide/pure_ast/ya.make @@ -1,6 +1,7 @@ LIBRARY() PEERDIR( + yql/essentials/parser/common/antlr4 yql/essentials/parser/antlr_ast/gen/v1_ansi_antlr4 yql/essentials/parser/antlr_ast/gen/v1_antlr4 ) diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp index 2d29d0a6ffd..71aa6aba9ef 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp +++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp @@ -10,9 +10,10 @@ namespace { class TParser: public NSQLTranslation::IParser { public: - explicit TParser(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TParser(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } @@ -23,39 +24,43 @@ public: NProtoAST::TProtoASTBuilder4< NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> - builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_); + builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); return builder.BuildAST(err); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; class TFactory: public NSQLTranslation::IParserFactory { public: - explicit TFactory(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TFactory(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { - return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_); + return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; } // namespace NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory( bool isAmbiguityError, - bool isAmbiguityDebugging) + bool isAmbiguityDebugging, + TMaybe<size_t> maxParseTreeDepth) { - return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging); + return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging, maxParseTreeDepth); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h index 146dcfff7aa..aeb83d7212f 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h +++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h @@ -1,10 +1,13 @@ #pragma once #include <yql/essentials/parser/proto_ast/common.h> +#include <util/generic/maybe.h> + namespace NSQLTranslationV1 { NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory( bool isAmbiguityError = false, - bool isAmbiguityDebugging = false); + bool isAmbiguityDebugging = false, + TMaybe<size_t> maxParseTreeDepth = Nothing()); } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp index 89eb7c5aa1a..d9aed46eae9 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp @@ -10,9 +10,10 @@ namespace { class TParser: public NSQLTranslation::IParser { public: - explicit TParser(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TParser(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } @@ -23,39 +24,43 @@ public: NProtoAST::TProtoASTBuilder4< NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> - builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_); + builder(query, queryName, arena, IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); return builder.BuildAST(err); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; class TFactory: public NSQLTranslation::IParserFactory { public: - explicit TFactory(bool isAmbuguityError, bool isAmbiguityDebugging) - : IsAmbiguityError_(isAmbuguityError) + TFactory(bool isAmbiguityError, bool isAmbiguityDebugging, TMaybe<size_t> maxParseTreeDepth) + : IsAmbiguityError_(isAmbiguityError) , IsAmbiguityDebugging_(isAmbiguityDebugging) + , MaxParseTreeDepth_(maxParseTreeDepth) { } std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { - return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_); + return std::make_unique<TParser>(IsAmbiguityError_, IsAmbiguityDebugging_, MaxParseTreeDepth_); } private: - bool IsAmbiguityError_; - bool IsAmbiguityDebugging_; + const bool IsAmbiguityError_; + const bool IsAmbiguityDebugging_; + const TMaybe<size_t> MaxParseTreeDepth_; }; } // namespace NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory( bool isAmbiguityError, - bool isAmbiguityDebugging) + bool isAmbiguityDebugging, + TMaybe<size_t> maxParseTreeDepth) { - return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging); + return MakeIntrusive<TFactory>(isAmbiguityError, isAmbiguityDebugging, maxParseTreeDepth); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h index 39ff5d6eba2..05a716c1c75 100644 --- a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h @@ -1,10 +1,13 @@ #pragma once #include <yql/essentials/parser/proto_ast/common.h> +#include <util/generic/maybe.h> + namespace NSQLTranslationV1 { NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory( bool isAmbiguityError = false, - bool isAmbiguityDebugging = false); + bool isAmbiguityDebugging = false, + TMaybe<size_t> maxParseTreeDepth = Nothing()); } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser_ut.cpp b/yql/essentials/sql/v1/proto_parser/proto_parser_ut.cpp new file mode 100644 index 00000000000..c45189d976a --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/proto_parser_ut.cpp @@ -0,0 +1,89 @@ +#include "proto_parser.h" + +#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> +#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> +#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> +#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h> + +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(ProtoParserTests) { + +google::protobuf::Message* SqlAST( + const TString& query, + NSQLTranslation::TTranslationSettings settings, + NYql::TIssues& issues) +{ + NSQLTranslationV1::TLexers lexers; + NSQLTranslationV1::TParsers parsers; + + if (!ParseTranslationSettings(query, settings, issues)) { + return nullptr; + } + + lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory(); + lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory(); + auto lexer = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer); + auto onNextToken = [&](NSQLTranslation::TParsedToken&& token) { + Y_UNUSED(token); + }; + + if (!lexer->Tokenize(query, "", onNextToken, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) { + return nullptr; + } + + parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth); + + parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory( + /*isAmbiguityError=*/false, + /*isAmbiguityDebugging=*/false, + settings.MaxParseTreeDepth); + + return NSQLTranslationV1::SqlAST( + parsers, + query, + /* queryName = */ "", + issues, + NSQLTranslation::SQL_MAX_PARSER_ERRORS, + settings.AnsiLexer, + settings.Arena); +} + +TString GenerateQuery(size_t depth = 8 * 1024) { + const TString prefix = "SELECT 1 FROM ("; + const TString core = "SELECT 1"; + const TString suffix = ")"; + + TString query; + query.reserve(prefix.size() * depth + core.size() + suffix.size() * depth); + for (size_t i = 0; i < depth; ++i) { + query += prefix; + } + query += core; + for (size_t i = 0; i < depth; ++i) { + query += suffix; + } + + return query; +} + +Y_UNIT_TEST(StackOverflowSubquery) { + const auto query = GenerateQuery(); + + google::protobuf::Arena arena; + NSQLTranslation::TTranslationSettings settings; + settings.Arena = &arena; + settings.MaxParseTreeDepth = 4 * 1024; + + NYql::TIssues issues; + const auto* m = SqlAST(query, settings, issues); + + UNIT_ASSERT(!m); + UNIT_ASSERT_STRING_CONTAINS(issues.ToOneLineString(), "Maximum parse tree depth exceeded"); +} + +} // Y_UNIT_TEST_SUITE(ProtoParserTests) diff --git a/yql/essentials/sql/v1/proto_parser/ut/ya.make b/yql/essentials/sql/v1/proto_parser/ut/ya.make new file mode 100644 index 00000000000..f52f74321bf --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/ut/ya.make @@ -0,0 +1,15 @@ +UNITTEST_FOR(yql/essentials/sql/v1/proto_parser) + +SRCS( + proto_parser_ut.cpp +) + +PEERDIR( + yql/essentials/sql/v1/lexer/antlr4 + yql/essentials/sql/v1/lexer/antlr4_ansi + yql/essentials/sql/v1/lexer + yql/essentials/sql/v1/proto_parser/antlr4 + yql/essentials/sql/v1/proto_parser/antlr4_ansi +) + +END() diff --git a/yql/essentials/sql/v1/proto_parser/ya.make b/yql/essentials/sql/v1/proto_parser/ya.make index 91047f7a32e..a1587d6aed5 100644 --- a/yql/essentials/sql/v1/proto_parser/ya.make +++ b/yql/essentials/sql/v1/proto_parser/ya.make @@ -20,3 +20,5 @@ RECURSE( antlr4 antlr4_ansi ) + +RECURSE_FOR_TESTS(ut) diff --git a/yql/essentials/sql/v1/select_yql.cpp b/yql/essentials/sql/v1/select_yql.cpp index 47e0b6e028b..c9b89286923 100644 --- a/yql/essentials/sql/v1/select_yql.cpp +++ b/yql/essentials/sql/v1/select_yql.cpp @@ -123,7 +123,7 @@ public: return new TYqlValuesNode(*this); } - bool SetColumns(TVector<TString> columns, TContext& ctx) { + bool SetColumns(TVector<TYqlColumnRef> columns, TContext& ctx) { if (columns.empty()) { return true; } @@ -142,14 +142,17 @@ private: TNodePtr BuildColumnList() const { TNodePtr columns = Y(); for (size_t i = 0; i < Width_; ++i) { + TPosition position = Pos_; TString name; if (!Columns_ || Columns_->size() <= i) { name = TStringBuilder() << "column" << i; } else { - name = Columns_->at(i); + const auto& c = Columns_->at(i); + position = c.Position; + name = c.Name; } - columns->Add(BuildQuotedAtom(Pos_, name)); + columns->Add(BuildQuotedAtom(std::move(position), name)); } return columns; } @@ -184,7 +187,7 @@ private: TNodePtr Values_; size_t Width_ = 0; - TMaybe<TVector<TString>> Columns_; + TMaybe<TVector<TYqlColumnRef>> Columns_; }; class TYqlSelectLikeNode: public INode { @@ -224,6 +227,11 @@ protected: }; class TYqlSetItemNode final: public TYqlSelectLikeNode, private TYqlSetItemArgs { + struct TProjectionItem { + TNodePtr Term; + bool IsAliasSynthetic = false; + }; + public: explicit TYqlSetItemNode(TYqlSetItemArgs&& args) : TYqlSelectLikeNode(args.Position) @@ -232,7 +240,9 @@ public: } bool DoInit(TContext& ctx, ISource* src) override { - if (!InitProjection(ctx, src) || + auto projection = InitProjection(ctx, src); + + if (!projection || !InitSource(ctx, src) || (Where && !Where->GetRef().Init(ctx, src)) || (GroupBy && !Init(ctx, src, *GroupBy)) || @@ -240,13 +250,14 @@ public: !Init(ctx, src, Windows) || !TYqlSelectLikeNode::Init(ctx, src, OrderBy) || (Limit && !Limit->GetRef().Init(ctx, src)) || - (Offset && !Offset->GetRef().Init(ctx, src))) { + (Offset && !Offset->GetRef().Init(ctx, src))) + { return false; } TNodePtr item = Y(); { - TNodePtr items = BuildYqlResultItems(Projection); + TNodePtr items = BuildYqlResultItems(*projection); if (!items) { return false; } @@ -354,31 +365,43 @@ public: } private: - bool InitProjection(TContext& ctx, ISource* src) const { + TMaybe<TVector<TProjectionItem>> InitProjection(TContext& ctx, ISource* src) const { return std::visit( TOverloaded{ [&](const TVector<TNodePtr>& terms) { + YQL_ENSURE(!terms.empty()); return InitTerms(ctx, src, terms); }, - [](const TPlainAsterisk&) { - return true; + [&](const TPlainAsterisk&) -> TMaybe<TVector<TProjectionItem>> { + return TVector<TProjectionItem>{}; }, }, Projection); } - bool InitTerms(TContext& ctx, ISource* src, const TVector<TNodePtr>& terms) const { - THashSet<TString> used = UsedLables(terms); + TMaybe<TVector<TProjectionItem>> + InitTerms(TContext& ctx, ISource* src, const TVector<TNodePtr>& terms) const { + THashSet<TString> used = UsedLabels(terms); + TVector<TProjectionItem> items(Reserve(terms.size())); for (size_t i = 0; i < terms.size(); ++i) { const TNodePtr& term = terms[i]; - TString label = TermAlias(term, i, used); + auto [label, isSyntheticA] = TermAlias(term, i, used); used.emplace(label); term->SetLabel(label); + + items.push_back({ + .Term = term, + .IsAliasSynthetic = isSyntheticA, + }); } - return ::NSQLTranslationV1::Init(ctx, src, terms); + if (!::NSQLTranslationV1::Init(ctx, src, terms)) { + return Nothing(); + } + + return items; } bool InitSource(TContext& ctx, ISource* src) const { @@ -457,7 +480,7 @@ private: return true; } - THashSet<TString> UsedLables(const TVector<TNodePtr>& terms) const { + THashSet<TString> UsedLabels(const TVector<TNodePtr>& terms) const { THashSet<TString> used(terms.size()); for (const TNodePtr& term : terms) { used.emplace(term->GetLabel()); @@ -465,46 +488,51 @@ private: return used; } - TString TermAlias(const TNodePtr& term, size_t i, const THashSet<TString>& used) const { + std::pair<TString, /*isSynthetic=*/bool> + TermAlias(const TNodePtr& term, size_t i, const THashSet<TString>& used) const { if (const TString& label = term->GetLabel(); !label.empty()) { - return label; + return {label, false}; } if (TMaybe<TString> alias = ColumnAlias(term)) { - return std::move(*alias); + return {std::move(*alias), false}; } for (;; ++i) { TString alias = TStringBuilder() << "column" << i; if (!used.contains(alias)) { - return alias; + return {alias, true}; } } } - TNodePtr BuildYqlResultItems(const TProjection& projection) const { - return std::visit( - TOverloaded{ - [&](const TVector<TNodePtr>& terms) { return BuildYqlResultItems(terms); }, - [&](const TPlainAsterisk& terms) { return BuildYqlResultItems(terms); }, - }, projection); - } + TNodePtr BuildYqlResultItems(const TVector<TProjectionItem>& projection) const { + if (projection.empty()) { + return BuildYqlResultItems(TPlainAsterisk()); + } - TNodePtr BuildYqlResultItems(const TVector<TNodePtr>& terms) const { TNodePtr items = Y(); - for (const TNodePtr& term : terms) { - items->Add(BuildYqlResultItem(term->GetLabel(), term)); + for (const auto& [term, isSynthetic] : projection) { + items->Add(BuildYqlResultItem(term->GetLabel(), isSynthetic, term)); } return items; } TNodePtr BuildYqlResultItems(const TPlainAsterisk&) const { - return Y(BuildYqlResultItem("", Y("YqlStar"))); + return Y(BuildYqlResultItem(/*name=*/"", /*isSynthetic=*/false, Y("YqlStar"))); } - TNodePtr BuildYqlResultItem(TString name, TNodePtr term) const { + TNodePtr BuildYqlResultItem(TString name, bool isSynthetic, TNodePtr term) const { TNodePtr nameAtom = BuildQuotedAtom(Pos_, name); - return Y("YqlResultItem", std::move(nameAtom), Y("Void"), Y("lambda", Q(Y()), std::move(term))); + + TNodePtr item = Y("YqlResultItem"); + item = L(std::move(item), std::move(nameAtom)); + item = L(std::move(item), Y("Void")); + if (isSynthetic) { + item = L(std::move(item), Q(Y(Q(Y(Q("synthetic")))))); + } + item = L(std::move(item), Y("lambda", Q(Y()), std::move(term))); + return item; } TMaybe<TString> ColumnAlias(const TNodePtr& term) const { @@ -524,16 +552,28 @@ private: } TMaybe<TNodePtr> BuildFromElement(TContext& ctx, const TYqlSource& source) const { - const auto build = [this](TNodePtr node, TString name, const TVector<TString>& columns) { + const auto build = [this](TNodePtr node, + TString name, + const TVector<TYqlColumnRef>& columns, + bool isCTE = false) + { YQL_ENSURE(!name.empty(), "An empty source name is unsupported"); + TNodePtr nameAtom = BuildQuotedAtom(Pos_, name); + TNodePtr columnList = Y(); - for (const TString& column : columns) { - columnList = L(std::move(columnList), BuildQuotedAtom(Pos_, column)); + for (const TYqlColumnRef& c : columns) { + columnList = L(std::move(columnList), BuildQuotedAtom(c.Position, c.Name)); } - TNodePtr nameAtom = BuildQuotedAtom(Pos_, name); - return Q(Y(std::move(node), std::move(nameAtom), Q(std::move(columnList)))); + TNodePtr x = Y(); + x = L(std::move(x), std::move(node)); + x = L(std::move(x), std::move(nameAtom)); + x = L(std::move(x), Q(std::move(columnList))); + if (isCTE) { + x = L(std::move(x), Q(Y(Q(Y(Q("cte")))))); + } + return Q(std::move(x)); }; if (!source.Alias) { @@ -550,7 +590,7 @@ private: } if (source.Alias->Kind == TYqlSourceAlias::EKind::CTE) { - return build(source.Node, source.Alias->Name, columns); + return build(source.Node, source.Alias->Name, columns, /*isCTE=*/true); } ctx.Error() << "Qualified by column names source alias " diff --git a/yql/essentials/sql/v1/select_yql.h b/yql/essentials/sql/v1/select_yql.h index 51debe6352f..1f9e58fb534 100644 --- a/yql/essentials/sql/v1/select_yql.h +++ b/yql/essentials/sql/v1/select_yql.h @@ -4,6 +4,11 @@ namespace NSQLTranslationV1 { +struct TYqlColumnRef { + TPosition Position; + TString Name; +}; + struct TYqlSourceAlias { enum class EKind { Subquery, @@ -12,7 +17,7 @@ struct TYqlSourceAlias { TPosition Position; TString Name; - TVector<TString> Columns; + TVector<TYqlColumnRef> Columns; EKind Kind = EKind::Subquery; }; diff --git a/yql/essentials/sql/v1/sql_select_yql.cpp b/yql/essentials/sql/v1/sql_select_yql.cpp index 0fc2d3f327c..369e4f715f9 100644 --- a/yql/essentials/sql/v1/sql_select_yql.cpp +++ b/yql/essentials/sql/v1/sql_select_yql.cpp @@ -143,7 +143,7 @@ private: return std::unexpected(ESQLError::Basic); } - TVector<TString> columns; + TVector<TYqlColumnRef> columns; if (rule.HasBlock2()) { columns = TableColumns(rule.GetBlock2().GetRule_pure_column_list1()); } @@ -568,6 +568,7 @@ private: } Token(rule.GetToken2()); + setItem.Position = Ctx_.Pos(); if (Mode_ != NSQLTranslation::ESqlMode::QUERY) { return Unsupported("ESqlMode != QUERY"); @@ -1547,15 +1548,23 @@ private: } } - TVector<TString> TableColumns(const TRule_pure_column_list& rule) { - TVector<TString> columns; - columns.emplace_back(Id(rule.GetRule_an_id2(), *this)); + TVector<TYqlColumnRef> TableColumns(const TRule_pure_column_list& rule) { + TVector<TYqlColumnRef> columns(Reserve(1 + rule.GetBlock3().size())); + columns.emplace_back(TableColumn(rule.GetRule_an_id2())); for (const auto& id : rule.GetBlock3()) { - columns.emplace_back(Id(id.GetRule_an_id2(), *this)); + columns.emplace_back(TableColumn(id.GetRule_an_id2())); } return columns; } + TYqlColumnRef TableColumn(const TRule_an_id& rule) { + TString id = Id(rule, *this); + return { + .Position = Ctx_.Pos(), + .Name = std::move(id), + }; + } + EYqlSetOp ToOp(const TRule_union_op& node) { const TString token = ToLowerUTF8(node.GetToken1().GetValue()); |