split lexers & parsers (#14899)

11 files changed, 124 insertions, 8 deletions

diff --git a/ydb/core/kqp/host/kqp_translate.cpp b/ydb/core/kqp/host/kqp_translate.cpp
index 4ff53e44e1..d3cf243463 100644
--- a/ydb/core/kqp/host/kqp_translate.cpp
+++ b/ydb/core/kqp/host/kqp_translate.cpp
@@ -4,6 +4,15 @@
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v0/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
+
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 #include <ydb/public/api/protos/ydb_query.pb.h>
 
@@ -332,9 +341,20 @@ TVector<TQueryAst> ParseStatements(const TString& queryText, bool isSql, TMaybe<
         NYql::TExprContext& ctx, TKqpTranslationSettingsBuilder& settingsBuilder) {
     TVector<TQueryAst> result;
     settingsBuilder.SetSqlVersion(sqlVersion);
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+    lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+    parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         NSQLTranslationV0::MakeTranslator(),
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/ydb/core/kqp/host/ya.make b/ydb/core/kqp/host/ya.make
index b0cb3f08c5..d3808abc29 100644
--- a/ydb/core/kqp/host/ya.make
+++ b/ydb/core/kqp/host/ya.make
@@ -24,6 +24,14 @@ PEERDIR(
     yql/essentials/sql
     yql/essentials/sql/v0
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+    yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
     yql/essentials/parser/pg_wrapper/interface
     yql/essentials/core
     yql/essentials/providers/common/codec
diff --git a/ydb/core/kqp/provider/rewrite_io_utils.cpp b/ydb/core/kqp/provider/rewrite_io_utils.cpp
index 5af07ab221..fdd5c84c7d 100644
--- a/ydb/core/kqp/provider/rewrite_io_utils.cpp
+++ b/ydb/core/kqp/provider/rewrite_io_utils.cpp
@@ -8,6 +8,14 @@
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/utils/log/log.h>
 
 namespace NYql {
@@ -27,9 +35,20 @@ TExprNode::TPtr CompileViewQuery(
     translationSettings.Mode = NSQLTranslation::ESqlMode::LIMITED_VIEW;
     NSQLTranslation::Deserialize(viewData.CapturedContext, translationSettings);
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+    lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+    parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         nullptr
     );
 
diff --git a/ydb/core/kqp/provider/ya.make b/ydb/core/kqp/provider/ya.make
index e33bedcb3e..b7ea1ebbf7 100644
--- a/ydb/core/kqp/provider/ya.make
+++ b/ydb/core/kqp/provider/ya.make
@@ -54,6 +54,14 @@ PEERDIR(
     yql/essentials/providers/result/provider
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+    yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
     ydb/library/ydb_issue/proto
     yql/essentials/public/issue
     yql/essentials/utils/log
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_ut.cpp b/ydb/core/kqp/provider/yql_kikimr_provider_ut.cpp
index d7989b6cdb..6c02ba35b9 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider_ut.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_provider_ut.cpp
@@ -16,7 +16,9 @@ TContext CreateDefaultParserContext(NYql::TIssues& issues) {
     settings.DefaultCluster = "/Cluster";
     settings.AssumeYdbOnClusterWithSlash = true;
 
-    return TContext(settings, { /* hints */ }, issues, { /* query */ });
+    NSQLTranslationV1::TLexers lexers;
+    NSQLTranslationV1::TParsers parsers;
+    return TContext(lexers, parsers, settings, { /* hints */ }, issues, { /* query */ });
 }
 
 TAstNode* CreateAlterTable(TContext& parserContext, const TString& tableName, const TAlterTableParameters& params) {
diff --git a/ydb/library/yql/tools/dqrun/dqrun.cpp b/ydb/library/yql/tools/dqrun/dqrun.cpp
index 5429fd358f..fd28ff6967 100644
--- a/ydb/library/yql/tools/dqrun/dqrun.cpp
+++ b/ydb/library/yql/tools/dqrun/dqrun.cpp
@@ -74,6 +74,10 @@
 #include <yql/essentials/core/pg_ext/yql_pg_ext.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 #include <yql/essentials/utils/log/tls_backend.h>
 #include <yql/essentials/utils/log/log.h>
@@ -1126,9 +1130,16 @@ int RunMain(int argc, const char* argv[])
         dataProvidersInit.push_back(GetDqDataProviderInitializer(&CreateDqExecTransformer, dqGateway, dqCompFactory, {}, storage));
     }
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/ydb/library/yql/tools/dqrun/ya.make b/ydb/library/yql/tools/dqrun/ya.make
index 775e43e5b0..2e8bb9042a 100644
--- a/ydb/library/yql/tools/dqrun/ya.make
+++ b/ydb/library/yql/tools/dqrun/ya.make
@@ -41,6 +41,10 @@ ENDIF()
         yql/essentials/core/services/mounts
         yql/essentials/sql
         yql/essentials/sql/v1
+        yql/essentials/sql/v1/lexer/antlr4
+        yql/essentials/sql/v1/lexer/antlr4_ansi
+        yql/essentials/sql/v1/proto_parser/antlr4
+        yql/essentials/sql/v1/proto_parser/antlr4_ansi
         ydb/library/yql/dq/actors/input_transforms
         ydb/library/yql/dq/comp_nodes
         ydb/library/yql/dq/opt
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/ya.make b/ydb/public/lib/ydb_cli/commands/interactive/ya.make
index 2f0f502e27..fe5459336a 100644
--- a/ydb/public/lib/ydb_cli/commands/interactive/ya.make
+++ b/ydb/public/lib/ydb_cli/commands/interactive/ya.make
@@ -11,6 +11,8 @@ PEERDIR(
     contrib/restricted/patched/replxx
     yql/essentials/parser/lexer_common
     yql/essentials/sql/v1/lexer
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
     yql/essentials/sql/settings
     yql/essentials/utils
     contrib/libs/antlr4_cpp_runtime
diff --git a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp b/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp
index 436fd27bec..08d5e6f24d 100644
--- a/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp
+++ b/ydb/public/lib/ydb_cli/commands/interactive/yql_highlight.cpp
@@ -5,6 +5,8 @@
 #include <yql/essentials/public/issue/yql_issue.h>
 #include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/sql/v1/lexer/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
 
 #include <util/charset/utf8.h>
 #include <util/string/strip.h>
@@ -26,6 +28,13 @@ namespace NYdb {
         using std::regex_constants::ECMAScript;
         using std::regex_constants::icase;
 
+        NSQLTranslationV1::TLexers MakeLexers() {
+            NSQLTranslationV1::TLexers lexers;
+            lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+            lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+            return lexers;
+        }
+
         constexpr const char* builtinFunctionPattern = ( //
             "^("
             "abs|aggregate_by|aggregate_list|aggregate_list_distinct|agg_list|agg_list_distinct|"
@@ -133,8 +142,8 @@ namespace NYdb {
             : Coloring(color)
             , BuiltinFunctionRegex(builtinFunctionPattern, ECMAScript | icase)
             , TypeRegex(typePattern, ECMAScript | icase)
-            , CppLexer(MakeLexer(/* ansi = */ false, /* antlr4 = */ true))
-            , ANSILexer(MakeLexer(/* ansi = */ true, /* antlr4 = */ true))
+            , CppLexer(MakeLexer(MakeLexers(), /* ansi = */ false, /* antlr4 = */ true))
+            , ANSILexer(MakeLexer(MakeLexers(), /* ansi = */ true, /* antlr4 = */ true))
         {
         }
 
diff --git a/ydb/public/lib/ydb_cli/dump/util/query_utils.cpp b/ydb/public/lib/ydb_cli/dump/util/query_utils.cpp
index 5acea79670..6c1a8c4414 100644
--- a/ydb/public/lib/ydb_cli/dump/util/query_utils.cpp
+++ b/ydb/public/lib/ydb_cli/dump/util/query_utils.cpp
@@ -5,6 +5,14 @@
 #include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/sql/v1/format/sql_format.h>
 #include <yql/essentials/sql/v1/proto_parser/proto_parser.h>
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/public/issue/yql_issue.h>
 
 #include <library/cpp/protobuf/util/simple_reflection.h>
@@ -237,9 +245,15 @@ bool SqlToProtoAst(const TString& queryStr, TRule_sql_query& queryProto, NYql::T
         return false;
     }
 
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+    parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     google::protobuf::Arena arena;
     const auto* parserProto = NSQLTranslationV1::SqlAST(
-        queryStr, "query", issues, 0, settings.AnsiLexer, settings.Antlr4Parser, settings.TestAntlr4, &arena
+        parsers, queryStr, "query", issues, 0, settings.AnsiLexer, settings.Antlr4Parser, &arena
     );
     if (!parserProto) {
         return false;
@@ -254,7 +268,18 @@ bool Format(const TString& query, TString& formattedQuery, NYql::TIssues& issues
     NSQLTranslation::TTranslationSettings settings;
     settings.Arena = &arena;
 
-    auto formatter = NSQLFormat::MakeSqlFormatter(settings);
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+    lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+    parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
+    auto formatter = NSQLFormat::MakeSqlFormatter(lexers, parsers, settings);
     return formatter->Format(query, formattedQuery, issues);
 }
 
diff --git a/ydb/public/lib/ydb_cli/dump/util/ya.make b/ydb/public/lib/ydb_cli/dump/util/ya.make
index e3375faedf..ffe6090574 100644
--- a/ydb/public/lib/ydb_cli/dump/util/ya.make
+++ b/ydb/public/lib/ydb_cli/dump/util/ya.make
@@ -16,6 +16,14 @@ PEERDIR(
     yql/essentials/sql/settings
     yql/essentials/sql/v1/format
     yql/essentials/sql/v1/proto_parser
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+    yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
     library/cpp/protobuf/util
 )