YQL-19594 Explicit lexers & parsers 2

commit_hash:2aa733486a55a04c39ff8bb1c7eaca3a85719647

31 files changed, 301 insertions, 41 deletions

diff --git a/yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp b/yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp
index 44445768a5..7e19a46369 100644
--- a/yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp
+++ b/yql/essentials/core/extract_predicate/ut/extract_predicate_ut.cpp
@@ -7,6 +7,10 @@
 #include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/ast/yql_ast_annotation.h>
 #include <yql/essentials/ast/yql_expr.h>
 #include <yql/essentials/core/cbo/simple/cbo_simple.h>
@@ -54,9 +58,16 @@ Y_UNIT_TEST_SUITE(TYqlExtractPredicate) {
         NSQLTranslation::TTranslationSettings settings;
         settings.SyntaxVersion = 1;
 
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+        lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+        NSQLTranslationV1::TParsers parsers;
+        parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+        parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
         NSQLTranslation::TTranslators translators(
             nullptr,
-            NSQLTranslationV1::MakeTranslator(),
+            NSQLTranslationV1::MakeTranslator(lexers, parsers),
             nullptr
         );
 
diff --git a/yql/essentials/core/extract_predicate/ut/ya.make b/yql/essentials/core/extract_predicate/ut/ya.make
index 8232ce6cbc..fbee19eb29 100644
--- a/yql/essentials/core/extract_predicate/ut/ya.make
+++ b/yql/essentials/core/extract_predicate/ut/ya.make
@@ -23,6 +23,10 @@ PEERDIR(
     yql/essentials/parser/pg_wrapper
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
 )
 
 YQL_LAST_ABI_VERSION()
diff --git a/yql/essentials/core/facade/ya.make b/yql/essentials/core/facade/ya.make
index 4e5a778518..756ed88de4 100644
--- a/yql/essentials/core/facade/ya.make
+++ b/yql/essentials/core/facade/ya.make
@@ -22,6 +22,16 @@ PEERDIR(
     yql/essentials/core/qplayer/udf_resolver
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+    #FIXME {
+    yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
+    #}
     yql/essentials/parser/pg_wrapper/interface
     yql/essentials/utils/log
     yql/essentials/core
diff --git a/yql/essentials/core/facade/yql_facade.cpp b/yql/essentials/core/facade/yql_facade.cpp
index 3051091e05..97a6f54d0b 100644
--- a/yql/essentials/core/facade/yql_facade.cpp
+++ b/yql/essentials/core/facade/yql_facade.cpp
@@ -12,6 +12,16 @@
 #include <yql/essentials/core/services/yql_eval_params.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+//FIXME {
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
+//}
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 #include <yql/essentials/utils/log/context.h>
 #include <yql/essentials/utils/log/profile.h>
@@ -731,9 +741,20 @@ bool TProgram::ParseSql(const NSQLTranslation::TTranslationSettings& settings)
     }
 
     currentSettings->EmitReadsForExists = true;
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+    lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+    parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/yql/essentials/core/services/mounts/ya.make b/yql/essentials/core/services/mounts/ya.make
index 0fd3bada7c..9701636fda 100644
--- a/yql/essentials/core/services/mounts/ya.make
+++ b/yql/essentials/core/services/mounts/ya.make
@@ -12,6 +12,10 @@ PEERDIR(
     yql/essentials/utils/log
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
 )
 
 YQL_LAST_ABI_VERSION()
diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp
index a8c8cd30df..d229b6d8f0 100644
--- a/yql/essentials/core/services/mounts/yql_mounts.cpp
+++ b/yql/essentials/core/services/mounts/yql_mounts.cpp
@@ -3,6 +3,10 @@
 #include <yql/essentials/core/yql_library_compiler.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/utils/log/profile.h>
 
 #include <library/cpp/resource/resource.h>
@@ -126,9 +130,15 @@ namespace NYql {
         TUserDataTable mounts;
         LoadYqlDefaultMounts(mounts);
 
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+        lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+        NSQLTranslationV1::TParsers parsers;
+        parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+        parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
         NSQLTranslation::TTranslators translators(
             nullptr,
-            NSQLTranslationV1::MakeTranslator(),
+            NSQLTranslationV1::MakeTranslator(lexers, parsers),
             nullptr
         );
 
diff --git a/yql/essentials/core/ut/yql_library_compiler_ut.cpp b/yql/essentials/core/ut/yql_library_compiler_ut.cpp
index d1c9356232..a9422e8a78 100644
--- a/yql/essentials/core/ut/yql_library_compiler_ut.cpp
+++ b/yql/essentials/core/ut/yql_library_compiler_ut.cpp
@@ -2,9 +2,6 @@
 
 #include "yql_library_compiler.h"
 
-#include <yql/essentials/sql/sql.h>
-#include <yql/essentials/sql/v1/sql.h>
-
 namespace NYql {
 
 Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
@@ -14,7 +11,7 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
     static bool CompileAndLink(const THashMap<TString, TString>& libs, TExprContext& ctx) {
         NSQLTranslation::TTranslators translators(
             nullptr,
-            NSQLTranslationV1::MakeTranslator(),
+            nullptr,
             nullptr
         );
 
@@ -36,7 +33,7 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
 
         NSQLTranslation::TTranslators translators(
             nullptr,
-            NSQLTranslationV1::MakeTranslator(),
+            nullptr,
             nullptr
         );
 
@@ -58,7 +55,7 @@ Y_UNIT_TEST_SUITE(TLibraryCompilerTests) {
 
         NSQLTranslation::TTranslators translators(
             nullptr,
-            NSQLTranslationV1::MakeTranslator(),
+            nullptr,
             nullptr
         );
 
diff --git a/yql/essentials/public/fastcheck/fastcheck.cpp b/yql/essentials/public/fastcheck/fastcheck.cpp
index deefd288e2..6f4274752b 100644
--- a/yql/essentials/public/fastcheck/fastcheck.cpp
+++ b/yql/essentials/public/fastcheck/fastcheck.cpp
@@ -7,15 +7,26 @@
 #include <yql/essentials/core/yql_user_data_storage.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 
 namespace NYql {
 namespace NFastCheck {
 
 bool CheckProgram(const TString& program, const TOptions& options, TIssues& errors) {
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/yql/essentials/public/fastcheck/translator.cpp b/yql/essentials/public/fastcheck/translator.cpp
index 8efbad60b4..eec337be86 100644
--- a/yql/essentials/public/fastcheck/translator.cpp
+++ b/yql/essentials/public/fastcheck/translator.cpp
@@ -1,5 +1,9 @@
 #include "check_runner.h"
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/sql/settings/translation_settings.h>
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 
@@ -75,7 +79,14 @@ private:
             return res;
         }
 
-        auto astRes = NSQLTranslationV1::SqlToYql(request.Program, settings);
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+        lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+        NSQLTranslationV1::TParsers parsers;
+        parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+        parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
+        auto astRes = NSQLTranslationV1::SqlToYql(lexers, parsers, request.Program, settings);
         res.Success = astRes.IsOk();
         res.Issues = astRes.Issues;
         return res;
diff --git a/yql/essentials/public/purecalc/common/interface.cpp b/yql/essentials/public/purecalc/common/interface.cpp
index 805fd1197b..c1a55d63a9 100644
--- a/yql/essentials/public/purecalc/common/interface.cpp
+++ b/yql/essentials/public/purecalc/common/interface.cpp
@@ -33,7 +33,7 @@ TProgramFactoryOptions::TProgramFactoryOptions()
     , NativeYtTypeFlags(0)
     , UseSystemColumns(false)
     , UseWorkerPool(true)
-    , UseAntlr4(false)
+    , UseAntlr4(true)
 {
 }
 
diff --git a/yql/essentials/public/purecalc/common/worker_factory.cpp b/yql/essentials/public/purecalc/common/worker_factory.cpp
index 9877429a9f..50b85ed05a 100644
--- a/yql/essentials/public/purecalc/common/worker_factory.cpp
+++ b/yql/essentials/public/purecalc/common/worker_factory.cpp
@@ -6,6 +6,18 @@
 
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+
+//FIXME {
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
+//}
+
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 #include <yql/essentials/ast/yql_expr.h>
 #include <yql/essentials/core/yql_expr_optimize.h>
@@ -208,9 +220,20 @@ TExprNode::TPtr TWorkerFactory<TBase>::Compile(
             }
         }
 
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+        lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
+        lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+        lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+        NSQLTranslationV1::TParsers parsers;
+        parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+        parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+        parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+        parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
         NSQLTranslation::TTranslators translators(
             nullptr,
-            NSQLTranslationV1::MakeTranslator(),
+            NSQLTranslationV1::MakeTranslator(lexers, parsers),
             NSQLTranslationPG::MakeTranslator()
         );
 
diff --git a/yql/essentials/public/purecalc/common/ya.make.inc b/yql/essentials/public/purecalc/common/ya.make.inc
index 0a81505642..4059a9e78d 100644
--- a/yql/essentials/public/purecalc/common/ya.make.inc
+++ b/yql/essentials/public/purecalc/common/ya.make.inc
@@ -39,6 +39,16 @@ PEERDIR(
     yql/essentials/core/type_ann
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+    #{ FIXME
+    yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
+    #}
     yql/essentials/parser/pg_wrapper/interface
     yql/essentials/providers/common/codec
     yql/essentials/providers/common/comp_nodes
diff --git a/yql/essentials/sql/v1/format/ya.make b/yql/essentials/sql/v1/format/ya.make
index 642addcb7b..a6d5ec0537 100644
--- a/yql/essentials/sql/v1/format/ya.make
+++ b/yql/essentials/sql/v1/format/ya.make
@@ -13,6 +13,7 @@ PEERDIR(
     yql/essentials/sql/settings
     yql/essentials/sql/v1/lexer
     yql/essentials/sql/v1/proto_parser
+    yql/essentials/parser/proto_ast/gen/v1_proto_split
     yql/essentials/core/sql_types
     library/cpp/protobuf/util
     library/cpp/resource
diff --git a/yql/essentials/sql/v1/lexer/lexer.cpp b/yql/essentials/sql/v1/lexer/lexer.cpp
index d477580b04..9a45971dbc 100644
--- a/yql/essentials/sql/v1/lexer/lexer.cpp
+++ b/yql/essentials/sql/v1/lexer/lexer.cpp
@@ -57,20 +57,12 @@ private:
                 return lexers.Antlr3;
             }
 
-            if (lexers.Antlr4) {
-                return lexers.Antlr4;
-            }
-
             return MakeDummyLexerFactory("antlr3");
         } else if (ansi && !antlr4) {
             if (lexers.Antlr3Ansi) {
                 return lexers.Antlr3Ansi;
             }
 
-            if (lexers.Antlr4Ansi) {
-                return lexers.Antlr4Ansi;
-            }
-
             return MakeDummyLexerFactory("antlr3_ansi");
         } else if (!ansi && antlr4) {
             if (lexers.Antlr4) {
diff --git a/yql/essentials/sql/v1/perf/parse.cpp b/yql/essentials/sql/v1/perf/parse.cpp
index 33174bb95c..0001b80ec4 100644
--- a/yql/essentials/sql/v1/perf/parse.cpp
+++ b/yql/essentials/sql/v1/perf/parse.cpp
@@ -1,4 +1,8 @@
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <util/datetime/cputimer.h>
 #include <util/string/builder.h>
@@ -29,7 +33,15 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES
     settings.MaxErrors = maxErrors;
     settings.Mode = mode;
     settings.Arena = &arena;
-    auto res = SqlToYql(query, settings);
+
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
+    auto res = SqlToYql(lexers, parsers, query, settings);
     if (debug == EDebugOutput::ToCerr) {
         Err2Str(res, debug);
     }
diff --git a/yql/essentials/sql/v1/perf/ya.make b/yql/essentials/sql/v1/perf/ya.make
index 99e9087682..dad0eeaee6 100644
--- a/yql/essentials/sql/v1/perf/ya.make
+++ b/yql/essentials/sql/v1/perf/ya.make
@@ -8,6 +8,10 @@ PEERDIR(
     yql/essentials/public/udf/service/exception_policy
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
     yql/essentials/sql/pg_dummy
 )
 
diff --git a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
index 81df851a92..7b22ae11e5 100644
--- a/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
+++ b/yql/essentials/sql/v1/proto_parser/proto_parser.cpp
@@ -52,11 +52,6 @@ google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query,
             if (!res) {
                 return res;
             }
-        } else if (parsers.Antlr4Ansi) {
-            res = parsers.Antlr4Ansi->MakeParser()->Parse(query, queryName, err, arena);
-            if (!res) {
-                return res;
-            }
         } else {
             ReportError(err, "antlr3_ansi");
             return nullptr;
@@ -70,11 +65,6 @@ google::protobuf::Message* SqlAST(const TParsers& parsers, const TString& query,
             if (!res) {
                 return res;
             }
-        } else if (parsers.Antlr4) {
-            res = parsers.Antlr4->MakeParser()->Parse(query, queryName, err, arena);
-            if (!res) {
-                return res;
-            }
         } else {
             ReportError(err, "antlr3");
             return nullptr;
diff --git a/yql/essentials/sql/v1/sql_translation.cpp b/yql/essentials/sql/v1/sql_translation.cpp
index f4416bb6c5..3e6f76d1ff 100644
--- a/yql/essentials/sql/v1/sql_translation.cpp
+++ b/yql/essentials/sql/v1/sql_translation.cpp
@@ -5178,7 +5178,7 @@ static TString GetLambdaText(TTranslation& ctx, TContext& Ctx, const TRule_lambd
 
     TVector<TString> statements;
     NYql::TIssues issues;
-    if (!SplitQueryToStatements(Ctx.Query, statements, issues, Ctx.Settings)) {
+    if (!SplitQueryToStatements(Ctx.Lexers, Ctx.Parsers, Ctx.Query, statements, issues, Ctx.Settings)) {
         return {};
     }
 
diff --git a/yql/essentials/sql/v1/sql_ut.cpp b/yql/essentials/sql/v1/sql_ut.cpp
index e8d1f3527b..aa1e17e53f 100644
--- a/yql/essentials/sql/v1/sql_ut.cpp
+++ b/yql/essentials/sql/v1/sql_ut.cpp
@@ -8178,13 +8178,18 @@ Y_UNIT_TEST_SUITE(QuerySplit) {
 
         NSQLTranslation::TTranslationSettings settings;
         settings.AnsiLexer = false;
-        settings.Antlr4Parser = true;
+        settings.Antlr4Parser = false;
         settings.Arena = &Arena;
 
         TVector<TString> statements;
         NYql::TIssues issues;
 
-        UNIT_ASSERT(NSQLTranslationV1::SplitQueryToStatements(query, statements, issues, settings));
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+        NSQLTranslationV1::TParsers parsers;
+        parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+
+        UNIT_ASSERT(NSQLTranslationV1::SplitQueryToStatements(lexers, parsers, query, statements, issues, settings));
 
         UNIT_ASSERT_VALUES_EQUAL(statements.size(), 3);
 
diff --git a/yql/essentials/sql/v1/sql_ut.h b/yql/essentials/sql/v1/sql_ut.h
index d01df1af2b..37e06e8708 100644
--- a/yql/essentials/sql/v1/sql_ut.h
+++ b/yql/essentials/sql/v1/sql_ut.h
@@ -2,6 +2,10 @@
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr3/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr3_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr3_ansi/proto_parser.h>
 #include <util/generic/map.h>
 
 #include <library/cpp/regex/pcre/pcre.h>
@@ -46,9 +50,16 @@ inline NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslat
     settings.Antlr4Parser = false;
     settings.SyntaxVersion = 1;
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr3 = NSQLTranslationV1::MakeAntlr3LexerFactory();
+    lexers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr3 = NSQLTranslationV1::MakeAntlr3ParserFactory();
+    parsers.Antlr3Ansi = NSQLTranslationV1::MakeAntlr3AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         nullptr
     );
 
diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.cpp b/yql/essentials/sql/v1/sql_ut_antlr4.cpp
index a150a7624b..6d54a4f409 100644
--- a/yql/essentials/sql/v1/sql_ut_antlr4.cpp
+++ b/yql/essentials/sql/v1/sql_ut_antlr4.cpp
@@ -8118,6 +8118,68 @@ Y_UNIT_TEST_SUITE(ColumnFamily) {
     }
 }
 
+Y_UNIT_TEST_SUITE(QuerySplit) {
+    Y_UNIT_TEST(Simple) {
+        TString query = R"(
+        ;
+        -- Comment 1
+        SELECT * From Input; -- Comment 2
+        -- Comment 3
+        $a = "a";
+
+        -- Comment 9
+        ;
+
+        -- Comment 10
+
+        -- Comment 8
+
+        $b = ($x) -> {
+        -- comment 4
+        return /* Comment 5 */ $x;
+        -- Comment 6
+        };
+
+        // Comment 7
+
+
+
+        )";
+
+        google::protobuf::Arena Arena;
+
+        NSQLTranslation::TTranslationSettings settings;
+        settings.AnsiLexer = false;
+        settings.Antlr4Parser = true;
+        settings.Arena = &Arena;
+
+        TVector<TString> statements;
+        NYql::TIssues issues;
+
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+        NSQLTranslationV1::TParsers parsers;
+        parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+
+        UNIT_ASSERT(NSQLTranslationV1::SplitQueryToStatements(lexers, parsers, query, statements, issues, settings));
+
+        UNIT_ASSERT_VALUES_EQUAL(statements.size(), 3);
+
+        UNIT_ASSERT_VALUES_EQUAL(statements[0], "-- Comment 1\n        SELECT * From Input; -- Comment 2\n");
+        UNIT_ASSERT_VALUES_EQUAL(statements[1], R"(-- Comment 3
+        $a = "a";)");
+        UNIT_ASSERT_VALUES_EQUAL(statements[2], R"(-- Comment 10
+
+        -- Comment 8
+
+        $b = ($x) -> {
+        -- comment 4
+        return /* Comment 5 */ $x;
+        -- Comment 6
+        };)");
+    }
+}
+
 Y_UNIT_TEST_SUITE(Transfer) {
     Y_UNIT_TEST(Lambda) {
         NYql::TAstParseResult res = SqlToYql(R"( use plato;
diff --git a/yql/essentials/sql/v1/sql_ut_antlr4.h b/yql/essentials/sql/v1/sql_ut_antlr4.h
index ab473e48d1..4f4203a789 100644
--- a/yql/essentials/sql/v1/sql_ut_antlr4.h
+++ b/yql/essentials/sql/v1/sql_ut_antlr4.h
@@ -2,6 +2,10 @@
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <util/generic/map.h>
 
 #include <library/cpp/testing/unittest/registar.h>
@@ -45,9 +49,16 @@ inline NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslat
     settings.Antlr4Parser = true;
     settings.SyntaxVersion = 1;
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         nullptr
     );
 
diff --git a/yql/essentials/sql/v1/ut/ya.make b/yql/essentials/sql/v1/ut/ya.make
index 6922d80243..7bb6f3d44b 100644
--- a/yql/essentials/sql/v1/ut/ya.make
+++ b/yql/essentials/sql/v1/ut/ya.make
@@ -13,6 +13,9 @@ PEERDIR(
     yql/essentials/sql/pg_dummy
     yql/essentials/sql/v1/format
     yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
 )
 
 TIMEOUT(300)
diff --git a/yql/essentials/sql/v1/ut_antlr4/ya.make b/yql/essentials/sql/v1/ut_antlr4/ya.make
index 427d68f825..ab7d04eea7 100644
--- a/yql/essentials/sql/v1/ut_antlr4/ya.make
+++ b/yql/essentials/sql/v1/ut_antlr4/ya.make
@@ -13,6 +13,15 @@ PEERDIR(
     yql/essentials/sql/pg_dummy
     yql/essentials/sql/v1/format
     yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+
+    #FIXME - switch sql_match_recognize_ut.cpp to antlr4
+    yql/essentials/sql/v1/lexer/antlr3
+    yql/essentials/sql/v1/lexer/antlr3_ansi
+    yql/essentials/sql/v1/proto_parser/antlr3
+    yql/essentials/sql/v1/proto_parser/antlr3_ansi
 )
 
 TIMEOUT(300)
diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp
index c7d8f68f55..bfe66837d4 100644
--- a/yql/essentials/tools/sql2yql/sql2yql.cpp
+++ b/yql/essentials/tools/sql2yql/sql2yql.cpp
@@ -229,9 +229,16 @@ int BuildAST(int argc, char* argv[]) {
         opts.PrintUsage(argv[0], Cerr);
     }
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
index ba0a10c311..7e9d364946 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
@@ -563,9 +563,16 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
     FuncRegistry_ = NKikimr::NMiniKQL::CreateFunctionRegistry(&NYql::NBacktrace::KikimrBackTrace,
         NKikimr::NMiniKQL::CreateBuiltinRegistry(), true, RunOptions_.UdfsPaths);
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/yql/tools/yqlrun/ya.make b/yql/tools/yqlrun/ya.make
index c83d16312a..d885e43c44 100644
--- a/yql/tools/yqlrun/ya.make
+++ b/yql/tools/yqlrun/ya.make
@@ -41,6 +41,10 @@ PEERDIR(
     yql/essentials/sql
     yql/essentials/sql/pg
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
 
     library/cpp/getopt
     library/cpp/logger
diff --git a/yql/tools/yqlrun/yqlrun.cpp b/yql/tools/yqlrun/yqlrun.cpp
index 936357fe8b..6ed7689a75 100644
--- a/yql/tools/yqlrun/yqlrun.cpp
+++ b/yql/tools/yqlrun/yqlrun.cpp
@@ -25,6 +25,10 @@
 #include <yql/essentials/ast/yql_expr.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 
 #include <library/cpp/getopt/last_getopt.h>
 #include <library/cpp/logger/stream.h>
@@ -209,9 +213,16 @@ int RunUI(int argc, const char* argv[])
 
     CommonInit(res, udfResolverPath, udfResolverFilterSyscalls, udfsPaths, fileStorage, udfResolver, funcRegistry, udfIndex);
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );
 
diff --git a/yt/yql/providers/yt/provider/ut/ya.make b/yt/yql/providers/yt/provider/ut/ya.make
index d06eaa14f7..fe8661e5ac 100644
--- a/yt/yql/providers/yt/provider/ut/ya.make
+++ b/yt/yql/providers/yt/provider/ut/ya.make
@@ -29,6 +29,10 @@ PEERDIR(
     yql/essentials/providers/result/provider
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
     yql/essentials/minikql/invoke_builtins/llvm16
     yql/essentials/minikql/comp_nodes/llvm16
     yql/essentials/sql/pg
diff --git a/yt/yql/providers/yt/provider/ya.make b/yt/yql/providers/yt/provider/ya.make
index fa23697866..f46939c794 100644
--- a/yt/yql/providers/yt/provider/ya.make
+++ b/yt/yql/providers/yt/provider/ya.make
@@ -81,6 +81,10 @@ PEERDIR(
     yql/essentials/public/udf/tz
     yql/essentials/sql
     yql/essentials/sql/v1
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
     yql/essentials/parser/pg_wrapper/interface
     yql/essentials/utils
     yql/essentials/utils/log
diff --git a/yt/yql/providers/yt/provider/yql_yt_table_desc.cpp b/yt/yql/providers/yt/provider/yql_yt_table_desc.cpp
index 32c1946610..fdf2ce3543 100644
--- a/yt/yql/providers/yt/provider/yql_yt_table_desc.cpp
+++ b/yt/yql/providers/yt/provider/yql_yt_table_desc.cpp
@@ -8,6 +8,10 @@
 #include <yql/essentials/core/issue/yql_issue.h>
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/parser/pg_wrapper/interface/parser.h>
 #include <yql/essentials/utils/yql_panic.h>
 
@@ -182,9 +186,16 @@ TExprNode::TPtr CompileViewSql(const TString& provider, const TString& cluster,
         }
     }
 
+    NSQLTranslationV1::TLexers lexers;
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    NSQLTranslationV1::TParsers parsers;
+    parsers.Antlr4 = NSQLTranslationV1::MakeAntlr4ParserFactory();
+    parsers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiParserFactory();
+
     NSQLTranslation::TTranslators translators(
         nullptr,
-        NSQLTranslationV1::MakeTranslator(),
+        NSQLTranslationV1::MakeTranslator(lexers, parsers),
         NSQLTranslationPG::MakeTranslator()
     );