summaryrefslogtreecommitdiffstats
path: root/yql/essentials
diff options
context:
space:
mode:
authorvvvv <[email protected]>2025-02-18 14:49:48 +0300
committervvvv <[email protected]>2025-02-18 16:08:18 +0300
commit1213d16b7fd20d4255d2ebb709a1745efbfeb91b (patch)
treea1cd11d96b5abf0a0ec287c76c4f3cfe9b32a86e /yql/essentials
parent408888e6801333da2d97af0b27a1c4da4448b9e0 (diff)
Introduced lexer & parser interfaces
commit_hash:fee365c90a176dd33a967cee20994b21d530080c
Diffstat (limited to 'yql/essentials')
-rw-r--r--yql/essentials/parser/lexer_common/lexer.cpp49
-rw-r--r--yql/essentials/parser/lexer_common/lexer.h11
-rw-r--r--yql/essentials/parser/lexer_common/ya.make1
-rw-r--r--yql/essentials/parser/proto_ast/common.h21
-rw-r--r--yql/essentials/sql/v1/lexer/antlr3/lexer.cpp40
-rw-r--r--yql/essentials/sql/v1/lexer/antlr3/lexer.h8
-rw-r--r--yql/essentials/sql/v1/lexer/antlr3/ya.make14
-rw-r--r--yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp40
-rw-r--r--yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h8
-rw-r--r--yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make14
-rw-r--r--yql/essentials/sql/v1/lexer/antlr4/lexer.cpp36
-rw-r--r--yql/essentials/sql/v1/lexer/antlr4/lexer.h8
-rw-r--r--yql/essentials/sql/v1/lexer/antlr4/ya.make14
-rw-r--r--yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp36
-rw-r--r--yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h8
-rw-r--r--yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make14
-rw-r--r--yql/essentials/sql/v1/lexer/lexer.cpp96
-rw-r--r--yql/essentials/sql/v1/lexer/lexer.h13
-rw-r--r--yql/essentials/sql/v1/lexer/ya.make17
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp35
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h8
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr3/ya.make13
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp35
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h8
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make13
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp35
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h8
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr4/ya.make13
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp35
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h8
-rw-r--r--yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make13
-rw-r--r--yql/essentials/sql/v1/proto_parser/proto_parser.cpp188
-rw-r--r--yql/essentials/sql/v1/proto_parser/proto_parser.h17
-rw-r--r--yql/essentials/sql/v1/proto_parser/ya.make20
-rw-r--r--yql/essentials/sql/v1/sql.cpp19
-rw-r--r--yql/essentials/sql/v1/sql.h5
36 files changed, 766 insertions, 155 deletions
diff --git a/yql/essentials/parser/lexer_common/lexer.cpp b/yql/essentials/parser/lexer_common/lexer.cpp
new file mode 100644
index 00000000000..c93dd35bd48
--- /dev/null
+++ b/yql/essentials/parser/lexer_common/lexer.cpp
@@ -0,0 +1,49 @@
+#include "lexer.h"
+#include <yql/essentials/public/issue/yql_issue.h>
+
+#include <util/string/builder.h>
+
+namespace NSQLTranslation {
+
+namespace {
+
+class TDummyLexer : public ILexer {
+public:
+ TDummyLexer(const TString& name)
+ : Name_(name)
+ {}
+
+ bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+ Y_UNUSED(query);
+ Y_UNUSED(queryName);
+ Y_UNUSED(onNextToken);
+ Y_UNUSED(maxErrors);
+ issues.AddIssue(NYql::TIssue({}, TStringBuilder() << "Lexer " << Name_ << " is not supported"));
+ return false;
+ }
+
+private:
+ const TString Name_;
+};
+
+class TDummyFactory: public ILexerFactory {
+public:
+ TDummyFactory(const TString& name)
+ : Name_(name)
+ {}
+
+ ILexer::TPtr MakeLexer() const final {
+ return MakeHolder<TDummyLexer>(Name_);
+ }
+
+private:
+ const TString Name_;
+};
+
+}
+
+TLexerFactoryPtr MakeDummyLexerFactory(const TString& name) {
+ return MakeIntrusive<TDummyFactory>(name);
+}
+
+}
diff --git a/yql/essentials/parser/lexer_common/lexer.h b/yql/essentials/parser/lexer_common/lexer.h
index f5d96006813..4de7eedb836 100644
--- a/yql/essentials/parser/lexer_common/lexer.h
+++ b/yql/essentials/parser/lexer_common/lexer.h
@@ -39,5 +39,16 @@ using TParsedTokenList = TVector<TParsedToken>;
IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end);
bool Tokenize(ILexer& lexer, const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors);
+class ILexerFactory : public TThrRefBase {
+public:
+ virtual ~ILexerFactory() = default;
+
+ virtual ILexer::TPtr MakeLexer() const = 0;
+};
+
+using TLexerFactoryPtr = TIntrusivePtr<ILexerFactory>;
+
+TLexerFactoryPtr MakeDummyLexerFactory(const TString& name);
+
}
diff --git a/yql/essentials/parser/lexer_common/ya.make b/yql/essentials/parser/lexer_common/ya.make
index a7601be02f4..14573434c47 100644
--- a/yql/essentials/parser/lexer_common/ya.make
+++ b/yql/essentials/parser/lexer_common/ya.make
@@ -7,6 +7,7 @@ PEERDIR(
SRCS(
tokens.cpp
lexer.h
+ lexer.cpp
hints.cpp
hints.h
parse_hints_impl.cpp
diff --git a/yql/essentials/parser/proto_ast/common.h b/yql/essentials/parser/proto_ast/common.h
index 013f4722883..64017a5c449 100644
--- a/yql/essentials/parser/proto_ast/common.h
+++ b/yql/essentials/parser/proto_ast/common.h
@@ -65,5 +65,24 @@ namespace NProtoAST {
IOutputStream& Err;
TString Name;
};
-
} // namespace NProtoAST
+
+namespace NSQLTranslation {
+ class IParser {
+ public:
+ virtual ~IParser() = default;
+
+ virtual google::protobuf::Message* Parse(
+ const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+ google::protobuf::Arena* arena) = 0;
+ };
+
+ class IParserFactory : public TThrRefBase {
+ public:
+ virtual ~IParserFactory() = default;
+
+ virtual std::unique_ptr<IParser> MakeParser() const = 0;
+ };
+
+ using TParserFactoryPtr = TIntrusivePtr<IParserFactory>;
+} // namespace NSQLTranslation
diff --git a/yql/essentials/sql/v1/lexer/antlr3/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr3/lexer.cpp
new file mode 100644
index 00000000000..e2d23d1503d
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr3/lexer.cpp
@@ -0,0 +1,40 @@
+#include "lexer.h"
+#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
+#include <yql/essentials/public/issue/yql_issue.h>
+#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
+#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
+
+namespace NALPDefault {
+extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
+}
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TLexer: public NSQLTranslation::ILexer {
+public:
+ bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+ NYql::TIssues newIssues;
+ NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
+ NProtoAST::TLexerTokensCollector3<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName);
+ tokensCollector.CollectTokens(collector, onNextToken);
+ issues.AddIssues(newIssues);
+ return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+ }
+};
+
+class TFactory: public NSQLTranslation::ILexerFactory {
+public:
+ THolder<NSQLTranslation::ILexer> MakeLexer() const final {
+ return MakeHolder<TLexer>();
+ }
+};
+
+}
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3LexerFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr3/lexer.h b/yql/essentials/sql/v1/lexer/antlr3/lexer.h
new file mode 100644
index 00000000000..0c4fd35d0f6
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr3/lexer.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/lexer_common/lexer.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3LexerFactory();
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr3/ya.make b/yql/essentials/sql/v1/lexer/antlr3/ya.make
new file mode 100644
index 00000000000..8fd1537311c
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr3/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/parser/lexer_common
+ yql/essentials/public/issue
+ yql/essentials/parser/proto_ast/collect_issues
+ yql/essentials/parser/proto_ast/gen/v1
+)
+
+SRCS(
+ lexer.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp
new file mode 100644
index 00000000000..5b5c459f701
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.cpp
@@ -0,0 +1,40 @@
+#include "lexer.h"
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
+#include <yql/essentials/public/issue/yql_issue.h>
+#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
+#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
+
+namespace NALPAnsi {
+extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
+}
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TLexer: public NSQLTranslation::ILexer {
+public:
+ bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+ NYql::TIssues newIssues;
+ NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
+ NProtoAST::TLexerTokensCollector3<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName);
+ tokensCollector.CollectTokens(collector, onNextToken);
+ issues.AddIssues(newIssues);
+ return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+ }
+};
+
+class TFactory: public NSQLTranslation::ILexerFactory {
+public:
+ THolder<NSQLTranslation::ILexer> MakeLexer() const final {
+ return MakeHolder<TLexer>();
+ }
+};
+
+}
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3AnsiLexerFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h
new file mode 100644
index 00000000000..c2347d82332
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/lexer_common/lexer.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr3AnsiLexerFactory();
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make b/yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make
new file mode 100644
index 00000000000..ed34a4fc28c
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr3_ansi/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/parser/lexer_common
+ yql/essentials/public/issue
+ yql/essentials/parser/proto_ast/collect_issues
+ yql/essentials/parser/proto_ast/gen/v1_ansi
+)
+
+SRCS(
+ lexer.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/lexer/antlr4/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr4/lexer.cpp
new file mode 100644
index 00000000000..5add4fc6bfb
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr4/lexer.cpp
@@ -0,0 +1,36 @@
+#include "lexer.h"
+#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
+#include <yql/essentials/public/issue/yql_issue.h>
+#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
+#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TLexer: public NSQLTranslation::ILexer {
+public:
+ bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+ NYql::TIssues newIssues;
+ NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
+ NProtoAST::TLexerTokensCollector4<NALPDefaultAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName);
+ tokensCollector.CollectTokens(collector, onNextToken);
+ issues.AddIssues(newIssues);
+ return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+ }
+};
+
+class TFactory: public NSQLTranslation::ILexerFactory {
+public:
+ THolder<NSQLTranslation::ILexer> MakeLexer() const final {
+ return MakeHolder<TLexer>();
+ }
+};
+
+}
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr4LexerFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr4/lexer.h b/yql/essentials/sql/v1/lexer/antlr4/lexer.h
new file mode 100644
index 00000000000..89d598d018e
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr4/lexer.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/lexer_common/lexer.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr4LexerFactory();
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr4/ya.make b/yql/essentials/sql/v1/lexer/antlr4/ya.make
new file mode 100644
index 00000000000..5b97daed5c6
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr4/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/parser/lexer_common
+ yql/essentials/public/issue
+ yql/essentials/parser/proto_ast/collect_issues
+ yql/essentials/parser/proto_ast/gen/v1_antlr4
+)
+
+SRCS(
+ lexer.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp
new file mode 100644
index 00000000000..ca5842ae984
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.cpp
@@ -0,0 +1,36 @@
+#include "lexer.h"
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h>
+#include <yql/essentials/public/issue/yql_issue.h>
+#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
+#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TLexer: public NSQLTranslation::ILexer {
+public:
+ bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) final {
+ NYql::TIssues newIssues;
+ NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
+ NProtoAST::TLexerTokensCollector4<NALPAnsiAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName);
+ tokensCollector.CollectTokens(collector, onNextToken);
+ issues.AddIssues(newIssues);
+ return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+ }
+};
+
+class TFactory: public NSQLTranslation::ILexerFactory {
+public:
+ THolder<NSQLTranslation::ILexer> MakeLexer() const final {
+ return MakeHolder<TLexer>();
+ }
+};
+
+}
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr4AnsiLexerFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h
new file mode 100644
index 00000000000..4aed14ef017
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/lexer_common/lexer.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TLexerFactoryPtr MakeAntlr4AnsiLexerFactory();
+
+}
diff --git a/yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make b/yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make
new file mode 100644
index 00000000000..292094bcd13
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/antlr4_ansi/ya.make
@@ -0,0 +1,14 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/parser/lexer_common
+ yql/essentials/public/issue
+ yql/essentials/parser/proto_ast/collect_issues
+ yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4
+)
+
+SRCS(
+ lexer.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/lexer/lexer.cpp b/yql/essentials/sql/v1/lexer/lexer.cpp
index fb9ca54f358..d477580b042 100644
--- a/yql/essentials/sql/v1/lexer/lexer.cpp
+++ b/yql/essentials/sql/v1/lexer/lexer.cpp
@@ -1,14 +1,12 @@
#include "lexer.h"
#include <yql/essentials/public/issue/yql_issue.h>
-#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
-#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
-#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h>
-#include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/parser/lexer_common/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/settings/translation_settings.h>
#include <util/string/ascii.h>
#include <util/string/builder.h>
@@ -18,17 +16,17 @@
#include <util/system/mutex.h>
#endif
-namespace NALPDefault {
-extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
-}
+namespace NSQLTranslationV1 {
-namespace NALPAnsi {
-extern ANTLR_UINT8 *SQLv1ParserTokenNames[];
+TLexers MakeAllLexers() {
+ return TLexers {
+ .Antlr3 = MakeAntlr3LexerFactory(),
+ .Antlr3Ansi = MakeAntlr3AnsiLexerFactory(),
+ .Antlr4 = MakeAntlr4LexerFactory(),
+ .Antlr4Ansi = MakeAntlr4AnsiLexerFactory()
+ };
}
-
-namespace NSQLTranslationV1 {
-
namespace {
#if defined(_tsan_enabled_)
@@ -36,47 +34,71 @@ TMutex SanitizerSQLTranslationMutex;
#endif
using NSQLTranslation::ILexer;
+using NSQLTranslation::MakeDummyLexerFactory;
class TV1Lexer : public ILexer {
public:
- explicit TV1Lexer(bool ansi, bool antlr4)
- : Ansi(ansi), Antlr4(antlr4)
+ explicit TV1Lexer(const TLexers& lexers, bool ansi, bool antlr4)
+ : Factory(GetFactory(lexers, ansi, antlr4))
{
}
bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) override {
- NYql::TIssues newIssues;
#if defined(_tsan_enabled_)
TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
#endif
- NSQLTranslation::TErrorCollectorOverIssues collector(newIssues, maxErrors, queryName);
- if (Ansi && !Antlr4) {
- NProtoAST::TLexerTokensCollector3<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName);
- tokensCollector.CollectTokens(collector, onNextToken);
- } else if (!Ansi && !Antlr4) {
- NProtoAST::TLexerTokensCollector3<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName);
- tokensCollector.CollectTokens(collector, onNextToken);
- } else if (Ansi && Antlr4) {
- NProtoAST::TLexerTokensCollector4<NALPAnsiAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName);
- tokensCollector.CollectTokens(collector, onNextToken);
+ return Factory->MakeLexer()->Tokenize(query, queryName, onNextToken, issues, maxErrors);
+ }
+
+private:
+ static NSQLTranslation::TLexerFactoryPtr GetFactory(const TLexers& lexers, bool ansi, bool antlr4) {
+ if (!ansi && !antlr4) {
+ if (lexers.Antlr3) {
+ return lexers.Antlr3;
+ }
+
+ if (lexers.Antlr4) {
+ return lexers.Antlr4;
+ }
+
+ return MakeDummyLexerFactory("antlr3");
+ } else if (ansi && !antlr4) {
+ if (lexers.Antlr3Ansi) {
+ return lexers.Antlr3Ansi;
+ }
+
+ if (lexers.Antlr4Ansi) {
+ return lexers.Antlr4Ansi;
+ }
+
+ return MakeDummyLexerFactory("antlr3_ansi");
+ } else if (!ansi && antlr4) {
+ if (lexers.Antlr4) {
+ return lexers.Antlr4;
+ }
+
+ return MakeDummyLexerFactory("antlr4");
} else {
- NProtoAST::TLexerTokensCollector4<NALPDefaultAntlr4::SQLv1Antlr4Lexer> tokensCollector(query, queryName);
- tokensCollector.CollectTokens(collector, onNextToken);
- }
+ if (lexers.Antlr4Ansi) {
+ return lexers.Antlr4Ansi;
+ }
- issues.AddIssues(newIssues);
- return !AnyOf(newIssues.begin(), newIssues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
+ return MakeDummyLexerFactory("antlr4_ansi");
+ }
}
private:
- const bool Ansi;
- const bool Antlr4;
+ NSQLTranslation::TLexerFactoryPtr Factory;
};
} // namespace
NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4) {
- return NSQLTranslation::ILexer::TPtr(new TV1Lexer(ansi, antlr4));
+ return NSQLTranslation::ILexer::TPtr(new TV1Lexer(MakeAllLexers(), ansi, antlr4));
+}
+
+NSQLTranslation::ILexer::TPtr MakeLexer(const TLexers& lexers, bool ansi, bool antlr4) {
+ return NSQLTranslation::ILexer::TPtr(new TV1Lexer(lexers, ansi, antlr4));
}
bool IsProbablyKeyword(const NSQLTranslation::TParsedToken& token) {
diff --git a/yql/essentials/sql/v1/lexer/lexer.h b/yql/essentials/sql/v1/lexer/lexer.h
index 3ce780686c6..0e94cb10bfa 100644
--- a/yql/essentials/sql/v1/lexer/lexer.h
+++ b/yql/essentials/sql/v1/lexer/lexer.h
@@ -4,8 +4,21 @@
namespace NSQLTranslationV1 {
+struct TLexers {
+ NSQLTranslation::TLexerFactoryPtr Antlr3;
+ NSQLTranslation::TLexerFactoryPtr Antlr3Ansi;
+ NSQLTranslation::TLexerFactoryPtr Antlr4;
+ NSQLTranslation::TLexerFactoryPtr Antlr4Ansi;
+};
+
+//FIXME remove
+TLexers MakeAllLexers();
+
+//FIXME remove
NSQLTranslation::ILexer::TPtr MakeLexer(bool ansi, bool antlr4);
+NSQLTranslation::ILexer::TPtr MakeLexer(const TLexers& lexers, bool ansi, bool antlr4);
+
// "Probably" because YQL keyword can be an identifier
// depending on a query context. For example
// in SELECT * FROM group - group is an identifier, but
diff --git a/yql/essentials/sql/v1/lexer/ya.make b/yql/essentials/sql/v1/lexer/ya.make
index cd5eea1b700..f32e79a472c 100644
--- a/yql/essentials/sql/v1/lexer/ya.make
+++ b/yql/essentials/sql/v1/lexer/ya.make
@@ -2,10 +2,11 @@ LIBRARY()
PEERDIR(
yql/essentials/core/issue/protos
- yql/essentials/parser/proto_ast/gen/v1
- yql/essentials/parser/proto_ast/gen/v1_ansi
- yql/essentials/parser/proto_ast/gen/v1_antlr4
- yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4
+ yql/essentials/sql/settings
+ yql/essentials/sql/v1/lexer/antlr3
+ yql/essentials/sql/v1/lexer/antlr3_ansi
+ yql/essentials/sql/v1/lexer/antlr4
+ yql/essentials/sql/v1/lexer/antlr4_ansi
)
SRCS(
@@ -18,6 +19,14 @@ SUPPRESSIONS(
END()
+RECURSE(
+ antlr3
+ antlr3_ansi
+ antlr4
+ antlr4_ansi
+)
+
RECURSE_FOR_TESTS(
ut
)
+
diff --git a/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp
new file mode 100644
index 00000000000..2490ca449d1
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.cpp
@@ -0,0 +1,35 @@
+#include "proto_parser.h"
+#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
+#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
+#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Parser.h>
+#include <yql/essentials/utils/yql_panic.h>
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TParser : public NSQLTranslation::IParser {
+public:
+ google::protobuf::Message* Parse(
+ const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+ google::protobuf::Arena* arena) final {
+ YQL_ENSURE(arena);
+ NProtoAST::TProtoASTBuilder3<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena);
+ return builder.BuildAST(err);
+ }
+};
+
+class TFactory: public NSQLTranslation::IParserFactory {
+public:
+ std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final {
+ return std::make_unique<TParser>();
+ }
+};
+
+}
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr3ParserFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h
new file mode 100644
index 00000000000..5d40c2f1700
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/proto_ast/common.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr3ParserFactory();
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr3/ya.make b/yql/essentials/sql/v1/proto_parser/antlr3/ya.make
new file mode 100644
index 00000000000..1200caced99
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr3/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/utils
+ yql/essentials/parser/proto_ast/antlr3
+ yql/essentials/parser/proto_ast/gen/v1
+)
+
+SRCS(
+ proto_parser.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp
new file mode 100644
index 00000000000..102f5e187e1
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.cpp
@@ -0,0 +1,35 @@
+#include "proto_parser.h"
+#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Parser.h>
+#include <yql/essentials/utils/yql_panic.h>
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TParser : public NSQLTranslation::IParser {
+public:
+ google::protobuf::Message* Parse(
+ const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+ google::protobuf::Arena* arena) final {
+ YQL_ENSURE(arena);
+ NProtoAST::TProtoASTBuilder3<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena);
+ return builder.BuildAST(err);
+ }
+};
+
+class TFactory: public NSQLTranslation::IParserFactory {
+public:
+ std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final {
+ return std::make_unique<TParser>();
+ }
+};
+
+}
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr3AnsiParserFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h
new file mode 100644
index 00000000000..f03f1ea0a72
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/proto_ast/common.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr3AnsiParserFactory();
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make
new file mode 100644
index 00000000000..ce8b3f7917a
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr3_ansi/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/utils
+ yql/essentials/parser/proto_ast/antlr3
+ yql/essentials/parser/proto_ast/gen/v1_ansi
+)
+
+SRCS(
+ proto_parser.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp
new file mode 100644
index 00000000000..33ec032f6cf
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.cpp
@@ -0,0 +1,35 @@
+#include "proto_parser.h"
+#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Parser.h>
+#include <yql/essentials/utils/yql_panic.h>
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TParser : public NSQLTranslation::IParser {
+public:
+ google::protobuf::Message* Parse(
+ const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+ google::protobuf::Arena* arena) final {
+ YQL_ENSURE(arena);
+ NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
+ return builder.BuildAST(err);
+ }
+};
+
+class TFactory: public NSQLTranslation::IParserFactory {
+public:
+ std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final {
+ return std::make_unique<TParser>();
+ }
+};
+
+}
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h
new file mode 100644
index 00000000000..5fcb4606736
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/proto_ast/common.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr4ParserFactory();
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr4/ya.make b/yql/essentials/sql/v1/proto_parser/antlr4/ya.make
new file mode 100644
index 00000000000..769455a99b6
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr4/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/utils
+ yql/essentials/parser/proto_ast/antlr4
+ yql/essentials/parser/proto_ast/gen/v1_antlr4
+)
+
+SRCS(
+ proto_parser.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp
new file mode 100644
index 00000000000..fdc0cf64704
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.cpp
@@ -0,0 +1,35 @@
+#include "proto_parser.h"
+#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h>
+#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Parser.h>
+#include <yql/essentials/utils/yql_panic.h>
+
+namespace NSQLTranslationV1 {
+
+namespace {
+
+class TParser : public NSQLTranslation::IParser {
+public:
+ google::protobuf::Message* Parse(
+ const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+ google::protobuf::Arena* arena) final {
+ YQL_ENSURE(arena);
+ NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
+ return builder.BuildAST(err);
+ }
+};
+
+class TFactory: public NSQLTranslation::IParserFactory {
+public:
+ std::unique_ptr<NSQLTranslation::IParser> MakeParser() const final {
+ return std::make_unique<TParser>();
+ }
+};
+
+}
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory() {
+ return MakeIntrusive<TFactory>();
+}
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h
new file mode 100644
index 00000000000..15174f44d23
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h
@@ -0,0 +1,8 @@
+#pragma once
+#include <yql/essentials/parser/proto_ast/common.h>
+
+namespace NSQLTranslationV1 {
+
+NSQLTranslation::TParserFactoryPtr MakeAntlr4AnsiParserFactory();
+
+}
diff --git a/yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make
new file mode 100644
index 00000000000..32c8a641d2d
--- /dev/null
+++ b/yql/essentials/sql/v1/proto_parser/antlr4_ansi/ya.make
@@ -0,0 +1,13 @@
+LIBRARY()
+
+PEERDIR(
+ yql/essentials/utils
+ yql/essentials/parser/proto_ast/antlr4
+ yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4
+)
+
+SRCS(
+ proto_parser.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
index 94849187ffd..81df851a925 100644
--- a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
+++ b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
@@ -2,21 +2,14 @@
#include <yql/essentials/utils/yql_panic.h>
-#include <yql/essentials/parser/proto_ast/antlr3/proto_ast_antlr3.h>
-#include <yql/essentials/parser/proto_ast/antlr4/proto_ast_antlr4.h>
#include <yql/essentials/parser/proto_ast/collect_issues/collect_issues.h>
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1/SQLv1Parser.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_ansi/SQLv1Parser.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_antlr4/SQLv1Antlr4Parser.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Lexer.h>
-#include <yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4/SQLv1Antlr4Parser.h>
-
-#include <yql/essentials/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h>
-
-#include <library/cpp/protobuf/util/simple_reflection.h>
+
+
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
+
#include <util/generic/algorithm.h>
#if defined(_tsan_enabled_)
@@ -32,121 +25,110 @@ namespace NSQLTranslationV1 {
TMutex SanitizerSQLTranslationMutex;
#endif
-using namespace NSQLv1Generated;
-
-void ValidateMessagesImpl(const google::protobuf::Message* msg1, const google::protobuf::Message* msg2, bool hasNonAscii) {
- YQL_ENSURE(!msg1 == !msg2);
- if (!msg1) {
- return;
- }
-
- YQL_ENSURE(msg1->GetDescriptor() == msg2->GetDescriptor());
- const auto descr = msg1->GetDescriptor();
- if (descr == NSQLv1Generated::TToken::GetDescriptor()) {
- const auto& token1 = dynamic_cast<const NSQLv1Generated::TToken&>(*msg1);
- const auto& token2 = dynamic_cast<const NSQLv1Generated::TToken&>(*msg2);
- const bool isEof1 = token1.GetId() == Max<ui32>();
- const bool isEof2 = token2.GetId() == Max<ui32>();
- YQL_ENSURE(isEof1 == isEof2);
- YQL_ENSURE(token1.GetValue() == token2.GetValue());
- if (!isEof1) {
- YQL_ENSURE(token1.GetLine() == token2.GetLine());
- if (!hasNonAscii) {
- YQL_ENSURE(token1.GetColumn() == token2.GetColumn());
- }
- }
+namespace {
- return;
- }
+void ReportError(NProtoAST::IErrorCollector& err, const TString& name) {
+ err.Error(0, 0, TStringBuilder() << "Parser " << name << " is not supported");
+}
- for (int i = 0; i < descr->field_count(); ++i) {
- const NProtoBuf::FieldDescriptor* fd = descr->field(i);
- NProtoBuf::TConstField field1(*msg1, fd);
- NProtoBuf::TConstField field2(*msg2, fd);
- YQL_ENSURE(field1.IsMessage() == field2.IsMessage());
- if (field1.IsMessage()) {
- YQL_ENSURE(field1.Size() == field2.Size());
- for (size_t j = 0; j < field1.Size(); ++j) {
- ValidateMessagesImpl(field1.template Get<NProtoBuf::Message>(j), field2.template Get<NProtoBuf::Message>(j), hasNonAscii);
- }
- }
- }
}
-void ValidateMessages(const TString& query, const google::protobuf::Message* msg1, const google::protobuf::Message* msg2) {
- const bool hasNonAscii = AnyOf(query, [](char c) { return !isascii(c);});
- return ValidateMessagesImpl(msg1, msg2, hasNonAscii);
+google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName, TIssues& err,
+ size_t maxErrors, bool ansiLexer, bool anlr4Parser, google::protobuf::Arena* arena) {
+ NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, queryName);
+ return SqlAST(parsers, query, queryName, collector, ansiLexer, anlr4Parser, arena);
}
-google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, TIssues& err,
- size_t maxErrors, bool ansiLexer, bool anlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
+google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
+ bool ansiLexer, bool anlr4Parser, google::protobuf::Arena* arena) {
YQL_ENSURE(arena);
#if defined(_tsan_enabled_)
TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
#endif
- NSQLTranslation::TErrorCollectorOverIssues collector(err, maxErrors, queryName);
if (ansiLexer && !anlr4Parser) {
- NProtoAST::TProtoASTBuilder3<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena);
- auto res = builder.BuildAST(collector);
- if (testAntlr4) {
- NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- auto res2 = builder.BuildAST(collector);
- ValidateMessages(query, res, res2);
+ google::protobuf::Message* res = nullptr;
+ if (parsers.Antlr3Ansi) {
+ res = parsers.Antlr3Ansi->MakeParser()->Parse(query, queryName, err, arena);
+ if (!res) {
+ return res;
+ }
+ } else if (parsers.Antlr4Ansi) {
+ res = parsers.Antlr4Ansi->MakeParser()->Parse(query, queryName, err, arena);
+ if (!res) {
+ return res;
+ }
+ } else {
+ ReportError(err, "antlr3_ansi");
+ return nullptr;
}
return res;
} else if (!ansiLexer && !anlr4Parser) {
- NProtoAST::TProtoASTBuilder3<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena);
- auto res = builder.BuildAST(collector);
- if (testAntlr4) {
- NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- auto res2 = builder.BuildAST(collector);
- ValidateMessages(query, res, res2);
+ google::protobuf::Message* res = nullptr;
+ if (parsers.Antlr3) {
+ res = parsers.Antlr3->MakeParser()->Parse(query, queryName, err, arena);
+ if (!res) {
+ return res;
+ }
+ } else if (parsers.Antlr4) {
+ res = parsers.Antlr4->MakeParser()->Parse(query, queryName, err, arena);
+ if (!res) {
+ return res;
+ }
+ } else {
+ ReportError(err, "antlr3");
+ return nullptr;
}
return res;
} else if (ansiLexer && anlr4Parser) {
- NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- return builder.BuildAST(collector);
- } else {
- NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- return builder.BuildAST(collector);
- }
-}
-
-google::protobuf::Message* SqlAST(const TString& query, const TString& queryName, NProtoAST::IErrorCollector& err,
- bool ansiLexer, bool anlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
- YQL_ENSURE(arena);
-#if defined(_tsan_enabled_)
- TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
-#endif
- if (ansiLexer && !anlr4Parser) {
- NProtoAST::TProtoASTBuilder3<NALPAnsi::SQLv1Parser, NALPAnsi::SQLv1Lexer> builder(query, queryName, arena);
- auto res = builder.BuildAST(err);
- if (testAntlr4) {
- NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- auto res2 = builder.BuildAST(err);
- ValidateMessages(query, res, res2);
+ google::protobuf::Message* res = nullptr;
+ if (parsers.Antlr4Ansi) {
+ res = parsers.Antlr4Ansi->MakeParser()->Parse(query, queryName, err, arena);
+ if (!res) {
+ return res;
+ }
+ } else {
+ ReportError(err, "antlr4_ansi");
+ return nullptr;
}
return res;
- } else if (!ansiLexer && !anlr4Parser) {
- NProtoAST::TProtoASTBuilder3<NALPDefault::SQLv1Parser, NALPDefault::SQLv1Lexer> builder(query, queryName, arena);
- auto res = builder.BuildAST(err);
- if (testAntlr4) {
- NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- auto res2 = builder.BuildAST(err);
- ValidateMessages(query, res, res2);
+ } else {
+ google::protobuf::Message* res = nullptr;
+ if (parsers.Antlr4) {
+ res = parsers.Antlr4->MakeParser()->Parse(query, queryName, err, arena);
+ if (!res) {
+ return res;
+ }
+ } else {
+ ReportError(err, "antlr4");
+ return nullptr;
}
return res;
- } else if (ansiLexer && anlr4Parser) {
- NProtoAST::TProtoASTBuilder4<NALPAnsiAntlr4::SQLv1Antlr4Parser, NALPAnsiAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- return builder.BuildAST(err);
- } else {
- NProtoAST::TProtoASTBuilder4<NALPDefaultAntlr4::SQLv1Antlr4Parser, NALPDefaultAntlr4::SQLv1Antlr4Lexer> builder(query, queryName, arena);
- return builder.BuildAST(err);
}
}
+google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
+ NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
+ Y_UNUSED(testAntlr4);
+ return SqlAST(MakeAllParsers(), query, queryName, err, maxErrors, ansiLexer, antlr4Parser, arena);
+}
+
+google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
+ NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena) {
+ Y_UNUSED(testAntlr4);
+ return SqlAST(MakeAllParsers(), query, queryName, err, ansiLexer, antlr4Parser, arena);
+}
+
+TParsers MakeAllParsers() {
+ return TParsers {
+ .Antlr3 = MakeAntlr3ParserFactory(),
+ .Antlr3Ansi = MakeAntlr3AnsiParserFactory(),
+ .Antlr4 = MakeAntlr4ParserFactory(),
+ .Antlr4Ansi = MakeAntlr4AnsiParserFactory()
+ };
+}
+
} // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.h b/yql/essentials/sql/v1/proto_parser/proto_parser.h
index 14440953196..6e9ba07cac4 100644
--- a/yql/essentials/sql/v1/proto_parser/proto_parser.h
+++ b/yql/essentials/sql/v1/proto_parser/proto_parser.h
@@ -14,9 +14,24 @@ namespace NSQLTranslation {
namespace NSQLTranslationV1 {
- google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
+ struct TParsers {
+ NSQLTranslation::TParserFactoryPtr Antlr3;
+ NSQLTranslation::TParserFactoryPtr Antlr3Ansi;
+ NSQLTranslation::TParserFactoryPtr Antlr4;
+ NSQLTranslation::TParserFactoryPtr Antlr4Ansi;
+ };
+
+ //FIXME remove
+ TParsers MakeAllParsers();
+
+ //FIXME remove
+ google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena);
google::protobuf::Message* SqlAST(const TString& query, const TString& queryName,
NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, bool testAntlr4, google::protobuf::Arena* arena);
+ google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName,
+ NYql::TIssues& err, size_t maxErrors, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena);
+ google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query, const TString& queryName,
+ NProtoAST::IErrorCollector& err, bool ansiLexer, bool antlr4Parser, google::protobuf::Arena* arena);
} // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/proto_parser/ya.make b/yql/essentials/sql/v1/proto_parser/ya.make
index edb24868baf..1ef29d39cab 100644
--- a/yql/essentials/sql/v1/proto_parser/ya.make
+++ b/yql/essentials/sql/v1/proto_parser/ya.make
@@ -4,14 +4,12 @@ PEERDIR(
yql/essentials/utils
yql/essentials/ast
- yql/essentials/parser/proto_ast/antlr3
- yql/essentials/parser/proto_ast/antlr4
yql/essentials/parser/proto_ast/collect_issues
- yql/essentials/parser/proto_ast/gen/v1
- yql/essentials/parser/proto_ast/gen/v1_ansi
- yql/essentials/parser/proto_ast/gen/v1_proto_split
- yql/essentials/parser/proto_ast/gen/v1_antlr4
- yql/essentials/parser/proto_ast/gen/v1_ansi_antlr4
+
+ yql/essentials/sql/v1/proto_parser/antlr3
+ yql/essentials/sql/v1/proto_parser/antlr3_ansi
+ yql/essentials/sql/v1/proto_parser/antlr4
+ yql/essentials/sql/v1/proto_parser/antlr4_ansi
)
SRCS(
@@ -19,3 +17,11 @@ SRCS(
)
END()
+
+RECURSE(
+ antlr3
+ antlr3_ansi
+ antlr4
+ antlr4_ansi
+)
+
diff --git a/yql/essentials/sql/v1/sql.cpp b/yql/essentials/sql/v1/sql.cpp
index 2fc80a2fe8e..d8558b21ec7 100644
--- a/yql/essentials/sql/v1/sql.cpp
+++ b/yql/essentials/sql/v1/sql.cpp
@@ -277,8 +277,15 @@ bool SplitQueryToStatements(const TString& query, TVector<TString>& statements,
class TTranslator : public NSQLTranslation::ITranslator {
public:
+ TTranslator(const TLexers& lexers, const TParsers& parsers)
+ : Lexers_(lexers)
+ , Parsers_(parsers)
+ {
+ Y_UNUSED(Parsers_);
+ }
+
NSQLTranslation::ILexer::TPtr MakeLexer(const NSQLTranslation::TTranslationSettings& settings) final {
- return NSQLTranslationV1::MakeLexer(settings.AnsiLexer, settings.Antlr4Parser);
+ return NSQLTranslationV1::MakeLexer(Lexers_, settings.AnsiLexer, settings.Antlr4Parser);
}
NYql::TAstParseResult TextToAst(const TString& query, const NSQLTranslation::TTranslationSettings& settings,
@@ -302,10 +309,18 @@ public:
NYql::TWarningRules* warningRules, TVector<NYql::TStmtParseInfo>* stmtParseInfo) final {
return SqlToAstStatements(query, settings, warningRules, stmtParseInfo);
}
+
+private:
+ const TLexers Lexers_;
+ const TParsers Parsers_;
};
NSQLTranslation::TTranslatorPtr MakeTranslator() {
- return MakeIntrusive<TTranslator>();
+ return MakeIntrusive<TTranslator>(MakeAllLexers(), MakeAllParsers());
+}
+
+NSQLTranslation::TTranslatorPtr MakeTranslator(const TLexers& lexers, const TParsers& parsers) {
+ return MakeIntrusive<TTranslator>(lexers, parsers);
}
} // namespace NSQLTranslationV1
diff --git a/yql/essentials/sql/v1/sql.h b/yql/essentials/sql/v1/sql.h
index 7f0ded1a556..b1aaf500012 100644
--- a/yql/essentials/sql/v1/sql.h
+++ b/yql/essentials/sql/v1/sql.h
@@ -1,6 +1,7 @@
#pragma once
#include <yql/essentials/ast/yql_ast.h>
+#include <yql/essentials/parser/lexer_common/lexer.h>
#include <yql/essentials/parser/lexer_common/hints.h>
#include <yql/essentials/parser/proto_ast/common.h>
#include <yql/essentials/parser/proto_ast/gen/v1_proto_split/SQLv1Parser.pb.main.h>
@@ -8,6 +9,8 @@
#include <yql/essentials/public/issue/yql_issue_manager.h>
#include <yql/essentials/sql/settings/translation_settings.h>
#include <yql/essentials/sql/settings/translator.h>
+#include <yql/essentials/sql/v1/lexer/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/proto_parser.h>
#include <google/protobuf/message.h>
@@ -29,4 +32,6 @@ namespace NSQLTranslationV1 {
const NSQLTranslation::TTranslationSettings& settings);
NSQLTranslation::TTranslatorPtr MakeTranslator();
+
+ NSQLTranslation::TTranslatorPtr MakeTranslator(const TLexers& lexers, const TParsers& parsers);
} // namespace NSQLTranslationV1