diff options
author | vvvv <[email protected]> | 2025-02-18 14:49:48 +0300 |
---|---|---|
committer | vvvv <[email protected]> | 2025-02-18 16:08:18 +0300 |
commit | 1213d16b7fd20d4255d2ebb709a1745efbfeb91b (patch) | |
tree | a1cd11d96b5abf0a0ec287c76c4f3cfe9b32a86e /yql/essentials | |
parent | 408888e6801333da2d97af0b27a1c4da4448b9e0 (diff) |
Introduced lexer & parser interfaces
commit_hash:fee365c90a176dd33a967cee20994b21d530080c
Diffstat (limited to 'yql/essentials')
36 files changed, 766 insertions, 155 deletions
diff --git a/yql/essentials/parser/lexer_common/lexer.cpp b/yql/essentials/parser/lexer_common/lexer.cpp new file mode 100644 index 00000000000..c93dd35bd48 --- /dev/null +++ b/yql/essentials/parser/lexer_common/lexer.cpp @@ -0,0 +1,49 @@ +#include "lexer.h" +#include <yql/essentials/public/issue/yql_issue.h> + +#include <util/string/builder.h> + +namespace NSQLTranslation { + +namespace { + +class TDummyLexer : public ILexer { +public: + TDummyLexer(const TString& name) + : Name_(name) + {} + + bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final { + Y_UNUSED(query); + Y_UNUSED(queryName); + Y_UNUSED(onNextToken); + Y_UNUSED(maxErrors); + issues.AddIssue(NYql::TIssue({}, TStringBuilder() << "Lexer " << Name_ << " is not supported")); + return false; + } + +private: + const TString Name_; +}; + +class TDummyFactory: public ILexerFactory { +public: + TDummyFactory(const TString& name) + : Name_(name) + {} + + ILexer::TPtr MakeLexer() const final { + return MakeHolder<TDummyLexer>(Name_); + } + +private: + const TString Name_; +}; + +} + +TLexerFactoryPtr MakeDummyLexerFactory(const TString& name) { + return MakeIntrusive<TDummyFactory>(name); +} + +} diff --git a/yql/essentials/parser/lexer_common/lexer.h b/yql/essentials/parser/lexer_common/lexer.h index f5d96006813..4de7eedb836 100644 --- a/yql/essentials/parser/lexer_common/lexer.h +++ b/yql/essentials/parser/lexer_common/lexer.h @@ -39,5 +39,16 @@ using TParsedTokenList = TVector<TParsedToken>; IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end); bool Tokenize(ILexer& lexer, const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors); +class ILexerFactory : public TThrRefBase { +public: + virtual ~ILexerFactory() = default; + + virtual ILexer::TPtr MakeLexer() const = 0; +}; + +using TLexerFactoryPtr = TIntrusivePtr<ILexerFactory>; + +TLexerFactoryPtr MakeDummyLexerFactory(const TString& name); + } diff --git a/yql/essentials/parser/lexer_common/ya.make b/yql/essentials/parser/lexer_common/ya.make index a7601be02f4..14573434c47 100644 --- a/yql/essentials/parser/lexer_common/ya.make +++ b/yql/essentials/parser/lexer_common/ya.make @@ -7,6 +7,7 @@ PEERDIR( SRCS( tokens.cpp lexer.h + lexer.cpp hints.cpp hints.h parse_hints_impl.cpp diff --git a/yql/essentials/parser/proto_ast/common.h b/yql/essentials/parser/proto_ast/common.h index 013f4722883..64017a5c449 100644 --- a/yql/essentials/parser/proto_ast/common.h +++ b/yql/essentials/parser/proto_ast/common.h @@ -65,5 +65,24 @@ namespace NProtoAST { IOutputStream& Err; TString Name; }; - } // namespace NProtoAST + +namespace NSQLTranslation { + class IParser { + public: + virtual ~IParser() = default; + + virtual google::protobuf::Message* Parse( + const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, + google::protobuf::Arena* arena) = 0; + }; + + class IParserFactory : public TThrRefBase { + public: + virtual ~IParserFactory() = default; + + virtual std::unique_ptr<IParser> MakeParser() const = 0; + }; + + using TParserFactoryPtr = TIntrusivePtr<IParserFactory>; +} // namespace NSQLTranslation diff --git a/yql/essentials/sql/v1/lexer/antlr3/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr3/lexer.cpp new file mode 100644 index 00000000000..e2d23d1503d --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr3/lexer.cpp @@ -0,0 +1,40 @@ +#include "lexer.h" +#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h> +#include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h> +#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h> + +namespace NALPDefault { +extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; +} + +namespace NSQLTranslationV1 { + +namespace { + +class TLexer: public NSQLTranslation::ILexer { +public: + bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final { + NYql::TIssues newIssues; + NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName); + NProtoAST::TLexerTokensCollector3<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName); + tokensCollector.CollectTokens(collector, onNextToken); + issues.AddIssues(newIssues); + return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + } +}; + +class TFactory: public NSQLTranslation::ILexerFactory { +public: + THolder<NSQLTranslation::ILexer> MakeLexer() const final { + return MakeHolder<TLexer>(); + } +}; + +} + +NSQLTranslation::TLexerFactoryPtr MakeAntlr3LexerFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/lexer/antlr3/lexer.h b/yql/essentials/sql/v1/lexer/antlr3/lexer.h new file mode 100644 index 00000000000..0c4fd35d0f6 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr3/lexer.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/lexer_common/lexer.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TLexerFactoryPtr MakeAntlr3LexerFactory(); + +} diff --git a/yql/essentials/sql/v1/lexer/antlr3/ya.make b/yql/essentials/sql/v1/lexer/antlr3/ya.make new file mode 100644 index 00000000000..8fd1537311c --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr3/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +PEERDIR( + yql/essentials/parser/lexer_common + yql/essentials/public/issue + yql/essentials/parser/proto_ast/collect_issues + yql/essentials/parser/proto_ast/gen/v1 +) + +SRCS( + lexer.cpp +) + +END() diff --git a/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp new file mode 100644 index 00000000000..5b5c459f701 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp @@ -0,0 +1,40 @@ +#include "lexer.h" +#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h> +#include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h> +#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h> + +namespace NALPAnsi { +extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; +} + +namespace NSQLTranslationV1 { + +namespace { + +class TLexer: public NSQLTranslation::ILexer { +public: + bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final { + NYql::TIssues newIssues; + NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName); + NProtoAST::TLexerTokensCollector3<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName); + tokensCollector.CollectTokens(collector, onNextToken); + issues.AddIssues(newIssues); + return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + } +}; + +class TFactory: public NSQLTranslation::ILexerFactory { +public: + THolder<NSQLTranslation::ILexer> MakeLexer() const final { + return MakeHolder<TLexer>(); + } +}; + +} + +NSQLTranslation::TLexerFactoryPtr MakeAntlr3AnsiLexerFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h new file mode 100644 index 00000000000..c2347d82332 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/lexer_common/lexer.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TLexerFactoryPtr MakeAntlr3AnsiLexerFactory(); + +} diff --git a/yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make b/yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make new file mode 100644 index 00000000000..ed34a4fc28c --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +PEERDIR( + yql/essentials/parser/lexer_common + yql/essentials/public/issue + yql/essentials/parser/proto_ast/collect_issues + yql/essentials/parser/proto_ast/gen/v1_ansi +) + +SRCS( + lexer.cpp +) + +END() diff --git a/yql/essentials/sql/v1/lexer/antlr4/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr4/lexer.cpp new file mode 100644 index 00000000000..5add4fc6bfb --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr4/lexer.cpp @@ -0,0 +1,36 @@ +#include "lexer.h" +#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h> +#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h> + +namespace NSQLTranslationV1 { + +namespace { + +class TLexer: public NSQLTranslation::ILexer { +public: + bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final { + NYql::TIssues newIssues; + NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName); + NProtoAST::TLexerTokensCollector4<NALPDefaultAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName); + tokensCollector.CollectTokens(collector, onNextToken); + issues.AddIssues(newIssues); + return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + } +}; + +class TFactory: public NSQLTranslation::ILexerFactory { +public: + THolder<NSQLTranslation::ILexer> MakeLexer() const final { + return MakeHolder<TLexer>(); + } +}; + +} + +NSQLTranslation::TLexerFactoryPtr MakeAntlr4LexerFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/lexer/antlr4/lexer.h b/yql/essentials/sql/v1/lexer/antlr4/lexer.h new file mode 100644 index 00000000000..89d598d018e --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr4/lexer.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/lexer_common/lexer.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TLexerFactoryPtr MakeAntlr4LexerFactory(); + +} diff --git a/yql/essentials/sql/v1/lexer/antlr4/ya.make b/yql/essentials/sql/v1/lexer/antlr4/ya.make new file mode 100644 index 00000000000..5b97daed5c6 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr4/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +PEERDIR( + yql/essentials/parser/lexer_common + yql/essentials/public/issue + yql/essentials/parser/proto_ast/collect_issues + yql/essentials/parser/proto_ast/gen/v1_antlr4 +) + +SRCS( + lexer.cpp +) + +END() diff --git a/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp new file mode 100644 index 00000000000..ca5842ae984 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp @@ -0,0 +1,36 @@ +#include "lexer.h" +#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/public/issue/yql_issue.h> +#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h> +#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h> + +namespace NSQLTranslationV1 { + +namespace { + +class TLexer: public NSQLTranslation::ILexer { +public: + bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final { + NYql::TIssues newIssues; + NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName); + NProtoAST::TLexerTokensCollector4<NALPAnsiAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName); + tokensCollector.CollectTokens(collector, onNextToken); + issues.AddIssues(newIssues); + return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + } +}; + +class TFactory: public NSQLTranslation::ILexerFactory { +public: + THolder<NSQLTranslation::ILexer> MakeLexer() const final { + return MakeHolder<TLexer>(); + } +}; + +} + +NSQLTranslation::TLexerFactoryPtr MakeAntlr4AnsiLexerFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h new file mode 100644 index 00000000000..4aed14ef017 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/lexer_common/lexer.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TLexerFactoryPtr MakeAntlr4AnsiLexerFactory(); + +} diff --git a/yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make b/yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make new file mode 100644 index 00000000000..292094bcd13 --- /dev/null +++ b/yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +PEERDIR( + yql/essentials/parser/lexer_common + yql/essentials/public/issue + yql/essentials/parser/proto_ast/collect_issues + yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4 +) + +SRCS( + lexer.cpp +) + +END() diff --git a/yql/essentials/sql/v1/lexer/lexer.cpp b/yql/essentials/sql/v1/lexer/lexer.cpp index fb9ca54f358..d477580b042 100644 --- a/yql/essentials/sql/v1/lexer/lexer.cpp +++ b/yql/essentials/sql/v1/lexer/lexer.cpp @@ -1,14 +1,12 @@ #include "lexer.h" #include <yql/essentials/public/issue/yql_issue.h> -#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h> -#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h> -#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h> -#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> -#include <yql/essentials/sql/v1/sql.h> +#include <yql/essentials/parser/lexer_common/lexer.h> +#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h> +#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h> +#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h> +#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h> +#include <yql/essentials/sql/settings/translation_settings.h> #include <util/string/ascii.h> #include <util/string/builder.h> @@ -18,17 +16,17 @@ #include <util/system/mutex.h> #endif -namespace NALPDefault { -extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; -} +namespace NSQLTranslationV1 { -namespace NALPAnsi { -extern ANTLR_UINT8 *SQLv1ParserTokenNames[]; +TLexers MakeAllLexers() { + return TLexers { + .Antlr3 = MakeAntlr3LexerFactory(), + .Antlr3Ansi = MakeAntlr3AnsiLexerFactory(), + .Antlr4 = MakeAntlr4LexerFactory(), + .Antlr4Ansi = MakeAntlr4AnsiLexerFactory() + }; } - -namespace NSQLTranslationV1 { - namespace { #if defined(_tsan_enabled_) @@ -36,47 +34,71 @@ TMutex SanitizerSQLTranslationMutex; #endif using NSQLTranslation::ILexer; +using NSQLTranslation::MakeDummyLexerFactory; class TV1Lexer : public ILexer { public: - explicit TV1Lexer(bool ansi, bool antlr4) - : Ansi(ansi), Antlr4(antlr4) + explicit TV1Lexer(const TLexers& lexers, bool ansi, bool antlr4) + : Factory(GetFactory(lexers, ansi, antlr4)) { } bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) override { - NYql::TIssues newIssues; #if defined(_tsan_enabled_) TGuard<TMutex> grd(SanitizerSQLTranslationMutex); #endif - NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName); - if (Ansi && !Antlr4) { - NProtoAST::TLexerTokensCollector3<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName); - tokensCollector.CollectTokens(collector, onNextToken); - } else if (!Ansi && !Antlr4) { - NProtoAST::TLexerTokensCollector3<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName); - tokensCollector.CollectTokens(collector, onNextToken); - } else if (Ansi && Antlr4) { - NProtoAST::TLexerTokensCollector4<NALPAnsiAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName); - tokensCollector.CollectTokens(collector, onNextToken); + return Factory->MakeLexer()->Tokenize(query, queryName, onNextToken, issues, maxErrors); + } + +private: + static NSQLTranslation::TLexerFactoryPtr GetFactory(const TLexers& lexers, bool ansi, bool antlr4) { + if (!ansi && !antlr4) { + if (lexers.Antlr3) { + return lexers.Antlr3; + } + + if (lexers.Antlr4) { + return lexers.Antlr4; + } + + return MakeDummyLexerFactory("antlr3"); + } else if (ansi && !antlr4) { + if (lexers.Antlr3Ansi) { + return lexers.Antlr3Ansi; + } + + if (lexers.Antlr4Ansi) { + return lexers.Antlr4Ansi; + } + + return MakeDummyLexerFactory("antlr3_ansi"); + } else if (!ansi && antlr4) { + if (lexers.Antlr4) { + return lexers.Antlr4; + } + + return MakeDummyLexerFactory("antlr4"); } else { - NProtoAST::TLexerTokensCollector4<NALPDefaultAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName); - tokensCollector.CollectTokens(collector, onNextToken); - } + if (lexers.Antlr4Ansi) { + return lexers.Antlr4Ansi; + } - issues.AddIssues(newIssues); - return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; }); + return MakeDummyLexerFactory("antlr4_ansi"); + } } private: - const bool Ansi; - const bool Antlr4; + NSQLTranslation::TLexerFactoryPtr Factory; }; } // namespace NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4) { - return NSQLTranslation::ILexer::TPtr(new TV1Lexer(ansi, antlr4)); + return NSQLTranslation::ILexer::TPtr(new TV1Lexer(MakeAllLexers(), ansi, antlr4)); +} + +NSQLTranslation::ILexer::TPtr MakeLexer(const TLexers& lexers, bool ansi, bool antlr4) { + return NSQLTranslation::ILexer::TPtr(new TV1Lexer(lexers, ansi, antlr4)); } bool IsProbablyKeyword(const NSQLTranslation::TParsedToken& token) { diff --git a/yql/essentials/sql/v1/lexer/lexer.h b/yql/essentials/sql/v1/lexer/lexer.h index 3ce780686c6..0e94cb10bfa 100644 --- a/yql/essentials/sql/v1/lexer/lexer.h +++ b/yql/essentials/sql/v1/lexer/lexer.h @@ -4,8 +4,21 @@ namespace NSQLTranslationV1 { +struct TLexers { + NSQLTranslation::TLexerFactoryPtr Antlr3; + NSQLTranslation::TLexerFactoryPtr Antlr3Ansi; + NSQLTranslation::TLexerFactoryPtr Antlr4; + NSQLTranslation::TLexerFactoryPtr Antlr4Ansi; +}; + +//FIXME remove +TLexers MakeAllLexers(); + +//FIXME remove NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4); +NSQLTranslation::ILexer::TPtr MakeLexer(const TLexers& lexers, bool ansi, bool antlr4); + // "Probably" because YQL keyword can be an identifier // depending on a query context. For example // in SELECT * FROM group - group is an identifier, but diff --git a/yql/essentials/sql/v1/lexer/ya.make b/yql/essentials/sql/v1/lexer/ya.make index cd5eea1b700..f32e79a472c 100644 --- a/yql/essentials/sql/v1/lexer/ya.make +++ b/yql/essentials/sql/v1/lexer/ya.make @@ -2,10 +2,11 @@ LIBRARY() PEERDIR( yql/essentials/core/issue/protos - yql/essentials/parser/proto_ast/gen/v1 - yql/essentials/parser/proto_ast/gen/v1_ansi - yql/essentials/parser/proto_ast/gen/v1_antlr4 - yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4 + yql/essentials/sql/settings + yql/essentials/sql/v1/lexer/antlr3 + yql/essentials/sql/v1/lexer/antlr3_ansi + yql/essentials/sql/v1/lexer/antlr4 + yql/essentials/sql/v1/lexer/antlr4_ansi ) SRCS( @@ -18,6 +19,14 @@ SUPPRESSIONS( END() +RECURSE( + antlr3 + antlr3_ansi + antlr4 + antlr4_ansi +) + RECURSE_FOR_TESTS( ut ) + diff --git a/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp new file mode 100644 index 00000000000..2490ca449d1 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp @@ -0,0 +1,35 @@ +#include "proto_parser.h" +#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h> +#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h> +#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Parser.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NSQLTranslationV1 { + +namespace { + +class TParser : public NSQLTranslation::IParser { +public: + google::protobuf::Message* Parse( + const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, + google::protobuf::Arena* arena) final { + YQL_ENSURE(arena); + NProtoAST::TProtoASTBuilder3<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); + return builder.BuildAST(err); + } +}; + +class TFactory: public NSQLTranslation::IParserFactory { +public: + std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { + return std::make_unique<TParser>(); + } +}; + +} + +NSQLTranslation::TParserFactoryPtr MakeAntlr3ParserFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h new file mode 100644 index 00000000000..5d40c2f1700 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/proto_ast/common.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TParserFactoryPtr MakeAntlr3ParserFactory(); + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr3/ya.make b/yql/essentials/sql/v1/proto_parser/antlr3/ya.make new file mode 100644 index 00000000000..1200caced99 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr3/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +PEERDIR( + yql/essentials/utils + yql/essentials/parser/proto_ast/antlr3 + yql/essentials/parser/proto_ast/gen/v1 +) + +SRCS( + proto_parser.cpp +) + +END() diff --git a/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp new file mode 100644 index 00000000000..102f5e187e1 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp @@ -0,0 +1,35 @@ +#include "proto_parser.h" +#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h> +#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h> +#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Parser.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NSQLTranslationV1 { + +namespace { + +class TParser : public NSQLTranslation::IParser { +public: + google::protobuf::Message* Parse( + const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, + google::protobuf::Arena* arena) final { + YQL_ENSURE(arena); + NProtoAST::TProtoASTBuilder3<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); + return builder.BuildAST(err); + } +}; + +class TFactory: public NSQLTranslation::IParserFactory { +public: + std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { + return std::make_unique<TParser>(); + } +}; + +} + +NSQLTranslation::TParserFactoryPtr MakeAntlr3AnsiParserFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h new file mode 100644 index 00000000000..f03f1ea0a72 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/proto_ast/common.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TParserFactoryPtr MakeAntlr3AnsiParserFactory(); + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make new file mode 100644 index 00000000000..ce8b3f7917a --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +PEERDIR( + yql/essentials/utils + yql/essentials/parser/proto_ast/antlr3 + yql/essentials/parser/proto_ast/gen/v1_ansi +) + +SRCS( + proto_parser.cpp +) + +END() diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp new file mode 100644 index 00000000000..33ec032f6cf --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp @@ -0,0 +1,35 @@ +#include "proto_parser.h" +#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h> +#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Parser.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NSQLTranslationV1 { + +namespace { + +class TParser : public NSQLTranslation::IParser { +public: + google::protobuf::Message* Parse( + const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, + google::protobuf::Arena* arena) final { + YQL_ENSURE(arena); + NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); + return builder.BuildAST(err); + } +}; + +class TFactory: public NSQLTranslation::IParserFactory { +public: + std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { + return std::make_unique<TParser>(); + } +}; + +} + +NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h new file mode 100644 index 00000000000..5fcb4606736 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/proto_ast/common.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory(); + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/ya.make b/yql/essentials/sql/v1/proto_parser/antlr4/ya.make new file mode 100644 index 00000000000..769455a99b6 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr4/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +PEERDIR( + yql/essentials/utils + yql/essentials/parser/proto_ast/antlr4 + yql/essentials/parser/proto_ast/gen/v1_antlr4 +) + +SRCS( + proto_parser.cpp +) + +END() diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp new file mode 100644 index 00000000000..fdc0cf64704 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp @@ -0,0 +1,35 @@ +#include "proto_parser.h" +#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h> +#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> +#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Parser.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NSQLTranslationV1 { + +namespace { + +class TParser : public NSQLTranslation::IParser { +public: + google::protobuf::Message* Parse( + const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, + google::protobuf::Arena* arena) final { + YQL_ENSURE(arena); + NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); + return builder.BuildAST(err); + } +}; + +class TFactory: public NSQLTranslation::IParserFactory { +public: + std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final { + return std::make_unique<TParser>(); + } +}; + +} + +NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory() { + return MakeIntrusive<TFactory>(); +} + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h new file mode 100644 index 00000000000..15174f44d23 --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h @@ -0,0 +1,8 @@ +#pragma once +#include <yql/essentials/parser/proto_ast/common.h> + +namespace NSQLTranslationV1 { + +NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory(); + +} diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make new file mode 100644 index 00000000000..32c8a641d2d --- /dev/null +++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +PEERDIR( + yql/essentials/utils + yql/essentials/parser/proto_ast/antlr4 + yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4 +) + +SRCS( + proto_parser.cpp +) + +END() diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp index 94849187ffd..81df851a925 100644 --- a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp +++ b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp @@ -2,21 +2,14 @@ #include <yql/essentials/utils/yql_panic.h> -#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h> -#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h> #include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h> -#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Parser.h> -#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Parser.h> -#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Parser.h> -#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h> -#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Parser.h> - -#include <yql/essentials/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h> - -#include <library/cpp/protobuf/util/simple_reflection.h> + + +#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h> +#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h> +#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h> +#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h> + #include <util/generic/algorithm.h> #if defined(_tsan_enabled_) @@ -32,121 +25,110 @@ namespace NSQLTranslationV1 { TMutex SanitizerSQLTranslationMutex; #endif -using namespace NSQLv1Generated; - -void ValidateMessagesImpl(const google::protobuf::Message* msg1, const google::protobuf::Message* msg2, bool hasNonAscii) { - YQL_ENSURE(!msg1 == !msg2); - if (!msg1) { - return; - } - - YQL_ENSURE(msg1->GetDescriptor() == msg2->GetDescriptor()); - const auto descr = msg1->GetDescriptor(); - if (descr == NSQLv1Generated::TToken::GetDescriptor()) { - const auto& token1 = dynamic_cast<const NSQLv1Generated::TToken&>(*msg1); - const auto& token2 = dynamic_cast<const NSQLv1Generated::TToken&>(*msg2); - const bool isEof1 = token1.GetId() == Max<ui32>(); - const bool isEof2 = token2.GetId() == Max<ui32>(); - YQL_ENSURE(isEof1 == isEof2); - YQL_ENSURE(token1.GetValue() == token2.GetValue()); - if (!isEof1) { - YQL_ENSURE(token1.GetLine() == token2.GetLine()); - if (!hasNonAscii) { - YQL_ENSURE(token1.GetColumn() == token2.GetColumn()); - } - } +namespace { - return; - } +void ReportError(NProtoAST::IErrorCollector& err, const TString& name) { + err.Error(0, 0, TStringBuilder() << "Parser " << name << " is not supported"); +} - for (int i = 0; i < descr->field_count(); ++i) { - const NProtoBuf::FieldDescriptor* fd = descr->field(i); - NProtoBuf::TConstField field1(*msg1, fd); - NProtoBuf::TConstField field2(*msg2, fd); - YQL_ENSURE(field1.IsMessage() == field2.IsMessage()); - if (field1.IsMessage()) { - YQL_ENSURE(field1.Size() == field2.Size()); - for (size_t j = 0; j < field1.Size(); ++j) { - ValidateMessagesImpl(field1.template Get<NProtoBuf::Message>(j), field2.template Get<NProtoBuf::Message>(j), hasNonAscii); - } - } - } } -void ValidateMessages(const TString& query, const google::protobuf::Message* msg1, const google::protobuf::Message* msg2) { - const bool hasNonAscii = AnyOf(query, [](char c) { return !isascii(c);}); - return ValidateMessagesImpl(msg1, msg2, hasNonAscii); +google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName, TIssues& err, + size_t maxErrors, bool ansiLexer, bool anlr4Parser, google::protobuf::Arena* arena) { + NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, queryName); + return SqlAST(parsers, query, queryName, collector, ansiLexer, anlr4Parser, arena); } -google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, TIssues& err, - size_t maxErrors, bool ansiLexer, bool anlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) { +google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, + bool ansiLexer, bool anlr4Parser, google::protobuf::Arena* arena) { YQL_ENSURE(arena); #if defined(_tsan_enabled_) TGuard<TMutex> grd(SanitizerSQLTranslationMutex); #endif - NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, queryName); if (ansiLexer && !anlr4Parser) { - NProtoAST::TProtoASTBuilder3<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); - auto res = builder.BuildAST(collector); - if (testAntlr4) { - NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - auto res2 = builder.BuildAST(collector); - ValidateMessages(query, res, res2); + google::protobuf::Message* res = nullptr; + if (parsers.Antlr3Ansi) { + res = parsers.Antlr3Ansi->MakeParser()->Parse(query, queryName, err, arena); + if (!res) { + return res; + } + } else if (parsers.Antlr4Ansi) { + res = parsers.Antlr4Ansi->MakeParser()->Parse(query, queryName, err, arena); + if (!res) { + return res; + } + } else { + ReportError(err, "antlr3_ansi"); + return nullptr; } return res; } else if (!ansiLexer && !anlr4Parser) { - NProtoAST::TProtoASTBuilder3<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); - auto res = builder.BuildAST(collector); - if (testAntlr4) { - NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - auto res2 = builder.BuildAST(collector); - ValidateMessages(query, res, res2); + google::protobuf::Message* res = nullptr; + if (parsers.Antlr3) { + res = parsers.Antlr3->MakeParser()->Parse(query, queryName, err, arena); + if (!res) { + return res; + } + } else if (parsers.Antlr4) { + res = parsers.Antlr4->MakeParser()->Parse(query, queryName, err, arena); + if (!res) { + return res; + } + } else { + ReportError(err, "antlr3"); + return nullptr; } return res; } else if (ansiLexer && anlr4Parser) { - NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - return builder.BuildAST(collector); - } else { - NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - return builder.BuildAST(collector); - } -} - -google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, - bool ansiLexer, bool anlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) { - YQL_ENSURE(arena); -#if defined(_tsan_enabled_) - TGuard<TMutex> grd(SanitizerSQLTranslationMutex); -#endif - if (ansiLexer && !anlr4Parser) { - NProtoAST::TProtoASTBuilder3<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena); - auto res = builder.BuildAST(err); - if (testAntlr4) { - NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - auto res2 = builder.BuildAST(err); - ValidateMessages(query, res, res2); + google::protobuf::Message* res = nullptr; + if (parsers.Antlr4Ansi) { + res = parsers.Antlr4Ansi->MakeParser()->Parse(query, queryName, err, arena); + if (!res) { + return res; + } + } else { + ReportError(err, "antlr4_ansi"); + return nullptr; } return res; - } else if (!ansiLexer && !anlr4Parser) { - NProtoAST::TProtoASTBuilder3<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena); - auto res = builder.BuildAST(err); - if (testAntlr4) { - NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - auto res2 = builder.BuildAST(err); - ValidateMessages(query, res, res2); + } else { + google::protobuf::Message* res = nullptr; + if (parsers.Antlr4) { + res = parsers.Antlr4->MakeParser()->Parse(query, queryName, err, arena); + if (!res) { + return res; + } + } else { + ReportError(err, "antlr4"); + return nullptr; } return res; - } else if (ansiLexer && anlr4Parser) { - NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - return builder.BuildAST(err); - } else { - NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena); - return builder.BuildAST(err); } } +google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, + NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) { + Y_UNUSED(testAntlr4); + return SqlAST(MakeAllParsers(), query, queryName, err, maxErrors, ansiLexer, antlr4Parser, arena); +} + +google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, + NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) { + Y_UNUSED(testAntlr4); + return SqlAST(MakeAllParsers(), query, queryName, err, ansiLexer, antlr4Parser, arena); +} + +TParsers MakeAllParsers() { + return TParsers { + .Antlr3 = MakeAntlr3ParserFactory(), + .Antlr3Ansi = MakeAntlr3AnsiParserFactory(), + .Antlr4 = MakeAntlr4ParserFactory(), + .Antlr4Ansi = MakeAntlr4AnsiParserFactory() + }; +} + } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.h b/yql/essentials/sql/v1/proto_parser/proto_parser.h index 14440953196..6e9ba07cac4 100644 --- a/yql/essentials/sql/v1/proto_parser/proto_parser.h +++ b/yql/essentials/sql/v1/proto_parser/proto_parser.h @@ -14,9 +14,24 @@ namespace NSQLTranslation { namespace NSQLTranslationV1 { - google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, + struct TParsers { + NSQLTranslation::TParserFactoryPtr Antlr3; + NSQLTranslation::TParserFactoryPtr Antlr3Ansi; + NSQLTranslation::TParserFactoryPtr Antlr4; + NSQLTranslation::TParserFactoryPtr Antlr4Ansi; + }; + + //FIXME remove + TParsers MakeAllParsers(); + + //FIXME remove + google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena); google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena); + google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName, + NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena); + google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName, + NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena); } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/proto_parser/ya.make b/yql/essentials/sql/v1/proto_parser/ya.make index edb24868baf..1ef29d39cab 100644 --- a/yql/essentials/sql/v1/proto_parser/ya.make +++ b/yql/essentials/sql/v1/proto_parser/ya.make @@ -4,14 +4,12 @@ PEERDIR( yql/essentials/utils yql/essentials/ast - yql/essentials/parser/proto_ast/antlr3 - yql/essentials/parser/proto_ast/antlr4 yql/essentials/parser/proto_ast/collect_issues - yql/essentials/parser/proto_ast/gen/v1 - yql/essentials/parser/proto_ast/gen/v1_ansi - yql/essentials/parser/proto_ast/gen/v1_proto_split - yql/essentials/parser/proto_ast/gen/v1_antlr4 - yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4 + + yql/essentials/sql/v1/proto_parser/antlr3 + yql/essentials/sql/v1/proto_parser/antlr3_ansi + yql/essentials/sql/v1/proto_parser/antlr4 + yql/essentials/sql/v1/proto_parser/antlr4_ansi ) SRCS( @@ -19,3 +17,11 @@ SRCS( ) END() + +RECURSE( + antlr3 + antlr3_ansi + antlr4 + antlr4_ansi +) + diff --git a/yql/essentials/sql/v1/sql.cpp b/yql/essentials/sql/v1/sql.cpp index 2fc80a2fe8e..d8558b21ec7 100644 --- a/yql/essentials/sql/v1/sql.cpp +++ b/yql/essentials/sql/v1/sql.cpp @@ -277,8 +277,15 @@ bool SplitQueryToStatements(const TString& query, TVector<TString>& statements, class TTranslator : public NSQLTranslation::ITranslator { public: + TTranslator(const TLexers& lexers, const TParsers& parsers) + : Lexers_(lexers) + , Parsers_(parsers) + { + Y_UNUSED(Parsers_); + } + NSQLTranslation::ILexer::TPtr MakeLexer(const NSQLTranslation::TTranslationSettings& settings) final { - return NSQLTranslationV1::MakeLexer(settings.AnsiLexer, settings.Antlr4Parser); + return NSQLTranslationV1::MakeLexer(Lexers_, settings.AnsiLexer, settings.Antlr4Parser); } NYql::TAstParseResult TextToAst(const TString& query, const NSQLTranslation::TTranslationSettings& settings, @@ -302,10 +309,18 @@ public: NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo) final { return SqlToAstStatements(query, settings, warningRules, stmtParseInfo); } + +private: + const TLexers Lexers_; + const TParsers Parsers_; }; NSQLTranslation::TTranslatorPtr MakeTranslator() { - return MakeIntrusive<TTranslator>(); + return MakeIntrusive<TTranslator>(MakeAllLexers(), MakeAllParsers()); +} + +NSQLTranslation::TTranslatorPtr MakeTranslator(const TLexers& lexers, const TParsers& parsers) { + return MakeIntrusive<TTranslator>(lexers, parsers); } } // namespace NSQLTranslationV1 diff --git a/yql/essentials/sql/v1/sql.h b/yql/essentials/sql/v1/sql.h index 7f0ded1a556..b1aaf500012 100644 --- a/yql/essentials/sql/v1/sql.h +++ b/yql/essentials/sql/v1/sql.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/ast/yql_ast.h> +#include <yql/essentials/parser/lexer_common/lexer.h> #include <yql/essentials/parser/lexer_common/hints.h> #include <yql/essentials/parser/proto_ast/common.h> #include <yql/essentials/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h> @@ -8,6 +9,8 @@ #include <yql/essentials/public/issue/yql_issue_manager.h> #include <yql/essentials/sql/settings/translation_settings.h> #include <yql/essentials/sql/settings/translator.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> +#include <yql/essentials/sql/v1/proto_parser/proto_parser.h> #include <google/protobuf/message.h> @@ -29,4 +32,6 @@ namespace NSQLTranslationV1 { const NSQLTranslation::TTranslationSettings& settings); NSQLTranslation::TTranslatorPtr MakeTranslator(); + + NSQLTranslation::TTranslatorPtr MakeTranslator(const TLexers& lexers, const TParsers& parsers); } // namespace NSQLTranslationV1 |