YQL-19616 refactor test lexers from sql2yql, supported facade run tools

commit_hash:fb1727dda2b8c7d2ff42d4436c54cb7aa1ce4bc2

13 files changed, 189 insertions, 82 deletions

diff --git a/yql/essentials/core/services/mounts/yql_mounts.cpp b/yql/essentials/core/services/mounts/yql_mounts.cpp
index d229b6d8f01..95dfbf3672e 100644
--- a/yql/essentials/core/services/mounts/yql_mounts.cpp
+++ b/yql/essentials/core/services/mounts/yql_mounts.cpp
@@ -122,7 +122,8 @@ namespace NYql {
         const THashMap<TString, TString>& clusterMapping,
         const THashSet<TString>& sqlFlags,
         bool optimizeLibraries,
-        THolder<TExprContext> ownedCtx)
+        THolder<TExprContext> ownedCtx,
+        TModuleResolver::TModuleChecker moduleChecker)
     {
         YQL_PROFILE_FUNC(DEBUG);
         auto ctx = rawCtx ? rawCtx : ownedCtx.Get();
@@ -152,7 +153,7 @@ namespace NYql {
         }
 
         moduleResolver = std::make_shared<TModuleResolver>(translators, std::move(modulesTable), ctx->NextUniqueId,
-            clusterMapping, sqlFlags, optimizeLibraries, std::move(ownedCtx));
+            clusterMapping, sqlFlags, optimizeLibraries, std::move(ownedCtx), moduleChecker);
         return mounts;
     }
 
@@ -162,22 +163,25 @@ namespace NYql {
         const TVector<NUserData::TUserData>& userData,
         const THashMap<TString, TString>& clusterMapping,
         const THashSet<TString>& sqlFlags,
-        bool optimizeLibraries) {
-        return GetYqlModuleResolverImpl(&ctx, moduleResolver, userData, clusterMapping, sqlFlags, optimizeLibraries, nullptr);
+        bool optimizeLibraries,
+        TModuleResolver::TModuleChecker moduleChecker) {
+        return GetYqlModuleResolverImpl(&ctx, moduleResolver, userData, clusterMapping, sqlFlags, optimizeLibraries, nullptr, moduleChecker);
     }
 
     bool GetYqlDefaultModuleResolver(
         TExprContext& ctx,
         IModuleResolver::TPtr& moduleResolver,
         const THashMap<TString, TString>& clusterMapping,
-        bool optimizeLibraries) {
-        return !GetYqlModuleResolverImpl(&ctx, moduleResolver, {}, clusterMapping, {}, optimizeLibraries, nullptr).empty();
+        bool optimizeLibraries,
+        TModuleResolver::TModuleChecker moduleChecker) {
+        return !GetYqlModuleResolverImpl(&ctx, moduleResolver, {}, clusterMapping, {}, optimizeLibraries, nullptr, moduleChecker).empty();
     }
 
     bool GetYqlDefaultModuleResolverWithContext(
         IModuleResolver::TPtr& moduleResolver,
         const THashMap<TString, TString>& clusterMapping,
-        bool optimizeLibraries) {
-        return !GetYqlModuleResolverImpl(nullptr, moduleResolver, {}, clusterMapping, {}, optimizeLibraries, MakeHolder<TExprContext>()).empty();
+        bool optimizeLibraries,
+        TModuleResolver::TModuleChecker moduleChecker) {
+        return !GetYqlModuleResolverImpl(nullptr, moduleResolver, {}, clusterMapping, {}, optimizeLibraries, MakeHolder<TExprContext>(), moduleChecker).empty();
     }
 }
diff --git a/yql/essentials/core/services/mounts/yql_mounts.h b/yql/essentials/core/services/mounts/yql_mounts.h
index f289d57b998..7c0c296aa64 100644
--- a/yql/essentials/core/services/mounts/yql_mounts.h
+++ b/yql/essentials/core/services/mounts/yql_mounts.h
@@ -11,12 +11,14 @@ bool GetYqlDefaultModuleResolver(
         TExprContext& ctx,
         IModuleResolver::TPtr& moduleResolver,
         const THashMap<TString, TString>& clusterMapping = {},
-        bool optimizeLibraries = true);
+        bool optimizeLibraries = true,
+        TModuleResolver::TModuleChecker moduleChecker = {});
 
 bool GetYqlDefaultModuleResolverWithContext(
     IModuleResolver::TPtr& moduleResolver,
     const THashMap<TString, TString>& clusterMapping = {},
-    bool optimizeLibraries = true);
+    bool optimizeLibraries = true,
+    TModuleResolver::TModuleChecker moduleChecker = {});
 
 TUserDataTable GetYqlModuleResolver(
         TExprContext& ctx,
@@ -24,6 +26,7 @@ TUserDataTable GetYqlModuleResolver(
         const TVector<NUserData::TUserData>& userData,
         const THashMap<TString, TString>& clusterMapping,
         const THashSet<TString>& sqlFlags,
-        bool optimizeLibraries = true);
+        bool optimizeLibraries = true,
+        TModuleResolver::TModuleChecker moduleChecker = {});
 
 } // namespace NYql
diff --git a/yql/essentials/core/yql_type_annotation.cpp b/yql/essentials/core/yql_type_annotation.cpp
index ebc268bd694..dbe6f27ed03 100644
--- a/yql/essentials/core/yql_type_annotation.cpp
+++ b/yql/essentials/core/yql_type_annotation.cpp
@@ -537,6 +537,12 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu
         ctx.IssueManager.RaiseIssue(addSubIssues(std::move(issue), astRes.Issues));
     }
 
+    if (!sExpr && ModuleChecker) {
+        if (!ModuleChecker(query, fullName, ctx)) {
+            return false;
+        }
+    }
+
     TLibraryCohesion cohesion;
     if (!CompileExpr(*astRes.Root, cohesion, LibsContext)) {
         ctx.AddError(addSubIssues(TIssue(pos, TStringBuilder() << "Failed to compile: " << fullName), LibsContext.IssueManager.GetIssues()));
@@ -647,7 +653,7 @@ IModuleResolver::TPtr TModuleResolver::CreateMutableChild() const {
         throw yexception() << "Module resolver should not contain user data and URL loader";
     }
 
-    return std::make_shared<TModuleResolver>(Translators, &Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs, FileAliasPrefix);
+    return std::make_shared<TModuleResolver>(Translators, &Modules, LibsContext.NextUniqueId, ClusterMapping, SqlFlags, OptimizeLibraries, KnownPackages, Libs, FileAliasPrefix, ModuleChecker);
 }
 
 void TModuleResolver::SetFileAliasPrefix(TString&& prefix) {
diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h
index 401e01454ac..ab2db6e9fbd 100644
--- a/yql/essentials/core/yql_type_annotation.h
+++ b/yql/essentials/core/yql_type_annotation.h
@@ -41,15 +41,19 @@ public:
 
 class TModuleResolver : public IModuleResolver {
 public:
+    using TModuleChecker = std::function<bool(const TString& query, const TString& fileName, TExprContext& ctx)>;
+
     TModuleResolver(const NSQLTranslation::TTranslators& translators, TModulesTable&& modules,
         ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
-        const THashSet<TString>& sqlFlags, bool optimizeLibraries = true, THolder<TExprContext> ownedCtx = {})
+        const THashSet<TString>& sqlFlags, bool optimizeLibraries = true,
+        THolder<TExprContext> ownedCtx = {}, TModuleChecker moduleChecker = {})
         : Translators(translators)
         , OwnedCtx(std::move(ownedCtx))
         , LibsContext(nextUniqueId)
         , Modules(std::move(modules))
         , ClusterMapping(clusterMapping)
         , SqlFlags(sqlFlags)
+        , ModuleChecker(moduleChecker)
         , OptimizeLibraries(optimizeLibraries)
     {
         if (OwnedCtx) {
@@ -60,7 +64,7 @@ public:
     TModuleResolver(const NSQLTranslation::TTranslators& translators, const TModulesTable* parentModules,
         ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
         const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString,
-        THashMap<int, TLibraryCohesion>>& libs, const TString& fileAliasPrefix)
+        THashMap<int, TLibraryCohesion>>& libs, const TString& fileAliasPrefix, TModuleChecker moduleChecker)
         : Translators(translators)
         , ParentModules(parentModules)
         , LibsContext(nextUniqueId)
@@ -68,6 +72,7 @@ public:
         , Libs(libs)
         , ClusterMapping(clusterMapping)
         , SqlFlags(sqlFlags)
+        , ModuleChecker(moduleChecker)
         , OptimizeLibraries(optimizeLibraries)
         , FileAliasPrefix(fileAliasPrefix)
     {
@@ -102,6 +107,10 @@ public:
         SqlFlags = flags;
     }
 
+    void SetModuleChecker(TModuleChecker moduleChecker) {
+        ModuleChecker = moduleChecker;
+    }
+
     void RegisterPackage(const TString& package) override;
     bool SetPackageDefaultVersion(const TString& package, ui32 version) override;
     const TExportTable* GetModule(const TString& module) const override;
@@ -139,6 +148,7 @@ private:
     TModulesTable Modules;
     THashMap<TString, TString> ClusterMapping;
     THashSet<TString> SqlFlags;
+    TModuleChecker ModuleChecker;
     const bool OptimizeLibraries;
     THolder<TExprContext::TFreezeGuard> FreezeGuard;
     TString FileAliasPrefix;
diff --git a/yql/essentials/sql/v1/lexer/check/check_lexers.cpp b/yql/essentials/sql/v1/lexer/check/check_lexers.cpp
new file mode 100644
index 00000000000..d0cccefdc79
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/check/check_lexers.cpp
@@ -0,0 +1,82 @@
+#include "check_lexers.h"
+
+
+#include <yql/essentials/sql/settings/translation_settings.h>
+#include <yql/essentials/sql/v1/lexer/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h>
+#include <yql/essentials/sql/v1/lexer/regex/lexer.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
+#include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
+#include <yql/essentials/core/issue/yql_issue.h>
+
+#include <util/string/builder.h>
+
+namespace NSQLTranslationV1 {
+
+bool CheckLexers(NYql::TPosition pos, const TString& query, NYql::TIssues& issues) {
+    NSQLTranslationV1::TLexers lexers;
+    NSQLTranslation::TTranslationSettings settings;
+    if (!NSQLTranslation::ParseTranslationSettings(query, settings, issues)) {
+        return false;
+    }
+
+    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
+    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
+    lexers.Antlr4Pure = NSQLTranslationV1::MakeAntlr4PureLexerFactory();
+    lexers.Antlr4PureAnsi = NSQLTranslationV1::MakeAntlr4PureAnsiLexerFactory();
+    auto lexerMain = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Default);
+    auto lexerPure = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Pure);
+    auto lexerRegex = NSQLTranslationV1::MakeRegexLexerFactory(settings.AnsiLexer)->MakeLexer();
+    TVector<NSQLTranslation::TParsedToken> mainTokens;
+    if (!lexerMain->Tokenize(query, "", [&](auto token) { mainTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+        return false;
+    }
+
+    TVector<NSQLTranslation::TParsedToken> pureTokens;
+    if (!lexerPure->Tokenize(query, "", [&](auto token) { pureTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+        return false;
+    }
+
+    TVector<NSQLTranslation::TParsedToken> regexTokens;
+    if (!lexerRegex->Tokenize(query, "", [&](auto token) { regexTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+        return false;
+    }
+
+    bool hasErrors = false;
+    auto check = [&](const char* name, const TVector<NSQLTranslation::TParsedToken>& otherTokens) {
+        if (mainTokens.size() != otherTokens.size()) {
+            hasErrors = true;
+            issues.AddIssue(NYql::TIssue(pos, TStringBuilder () << "Mismatch token count, main: " <<
+                mainTokens.size() << ", " << name << ": " << otherTokens.size() << "\n"));
+        }
+
+        TStringBuilder textBuilder;
+
+        for (size_t i = 0; i < Min(mainTokens.size(), otherTokens.size()); ++i) {
+            if (mainTokens[i].Name != otherTokens[i].Name || mainTokens[i].Content != otherTokens[i].Content) {
+                hasErrors = true;
+                TStringBuilder err;
+                err << "Mismatch token #" << i << ", main: " << mainTokens[i].Name << ":" << mainTokens[i].Content
+                    << ", " << name << ": " << otherTokens[i].Name << ":" << otherTokens[i].Content << "\n";
+                err << "Text sample: [";
+                TString text = textBuilder;
+                constexpr size_t LexerContextSample = 50;
+                err << text.substr(text.size() >= LexerContextSample ? text.size() - LexerContextSample : 0u, LexerContextSample);
+                err << "]\n";
+                issues.AddIssue(NYql::TIssue(pos, err));
+                break;
+            }
+
+            textBuilder << mainTokens[i].Content;
+        }
+    };
+
+    check("pure", pureTokens);
+    check("regex", regexTokens);
+    return !hasErrors;
+}
+
+}
diff --git a/yql/essentials/sql/v1/lexer/check/check_lexers.h b/yql/essentials/sql/v1/lexer/check/check_lexers.h
new file mode 100644
index 00000000000..0fceaa2e0bd
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/check/check_lexers.h
@@ -0,0 +1,9 @@
+#pragma once
+#include <yql/essentials/core/issue/yql_issue.h>
+#include <util/generic/string.h>
+
+namespace NSQLTranslationV1 {
+
+bool CheckLexers(NYql::TPosition pos, const TString& query, NYql::TIssues& issues);
+
+}
diff --git a/yql/essentials/sql/v1/lexer/check/ya.make b/yql/essentials/sql/v1/lexer/check/ya.make
new file mode 100644
index 00000000000..dc4ac21836d
--- /dev/null
+++ b/yql/essentials/sql/v1/lexer/check/ya.make
@@ -0,0 +1,21 @@
+LIBRARY()
+
+SRCS(
+    check_lexers.h
+    check_lexers.cpp
+)
+
+PEERDIR(
+    yql/essentials/core/issue
+    yql/essentials/sql/settings
+    yql/essentials/sql/v1/lexer
+    yql/essentials/sql/v1/lexer/antlr4
+    yql/essentials/sql/v1/lexer/antlr4_ansi
+    yql/essentials/sql/v1/lexer/antlr4_pure
+    yql/essentials/sql/v1/lexer/antlr4_pure_ansi
+    yql/essentials/sql/v1/lexer/regex
+    yql/essentials/sql/v1/proto_parser/antlr4
+    yql/essentials/sql/v1/proto_parser/antlr4_ansi
+)
+
+END()
diff --git a/yql/essentials/sql/v1/lexer/ya.make b/yql/essentials/sql/v1/lexer/ya.make
index 66c0c87f15f..6462ced1991 100644
--- a/yql/essentials/sql/v1/lexer/ya.make
+++ b/yql/essentials/sql/v1/lexer/ya.make
@@ -22,6 +22,7 @@ RECURSE(
     antlr4_ansi
     antlr4_pure
     antlr4_pure_ansi
+    check
     regex
 )
 
diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp
index 8d185f72bb0..c3ffc588a13 100644
--- a/yql/essentials/tools/sql2yql/sql2yql.cpp
+++ b/yql/essentials/tools/sql2yql/sql2yql.cpp
@@ -6,11 +6,9 @@
 
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/check/check_lexers.h>
 #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
 #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
-#include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h>
-#include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h>
-#include <yql/essentials/sql/v1/lexer/regex/lexer.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
@@ -155,71 +153,14 @@ bool TestFormat(
     return true;
 }
 
-bool TestLexers(
-    const TString& query
-) {
-    NSQLTranslationV1::TLexers lexers;
-    NSQLTranslation::TTranslationSettings settings;
+bool TestLexers(const TString& query) {
     NYql::TIssues issues;
-    if (!NSQLTranslation::ParseTranslationSettings(query, settings, issues)) {
-        Cerr << issues.ToString();
-        return false;
-    }
-
-    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
-    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
-    lexers.Antlr4Pure = NSQLTranslationV1::MakeAntlr4PureLexerFactory();
-    lexers.Antlr4PureAnsi = NSQLTranslationV1::MakeAntlr4PureAnsiLexerFactory();
-    auto lexerMain = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Default);
-    auto lexerPure = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Pure);
-    auto lexerRegex = NSQLTranslationV1::MakeRegexLexerFactory(settings.AnsiLexer)->MakeLexer();
-    TVector<NSQLTranslation::TParsedToken> mainTokens;
-    if (!lexerMain->Tokenize(query, "", [&](auto token) { mainTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
-        Cerr << issues.ToString();
-        return false;
-    }
-
-    TVector<NSQLTranslation::TParsedToken> pureTokens;
-    if (!lexerPure->Tokenize(query, "", [&](auto token) { pureTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+    if (!NSQLTranslationV1::CheckLexers({}, query, issues)) {
         Cerr << issues.ToString();
         return false;
     }
 
-    TVector<NSQLTranslation::TParsedToken> regexTokens;
-    if (!lexerRegex->Tokenize(query, "", [&](auto token) { regexTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
-        Cerr << issues.ToString();
-        return false;
-    }
-
-    bool hasErrors = false;
-    auto check = [&](const char* name, const TVector<NSQLTranslation::TParsedToken>& otherTokens) {
-        if (mainTokens.size() != otherTokens.size()) {
-            hasErrors = true;
-            Cerr << "Mismatch token count, main: " << mainTokens.size() << ", " << name << ": " << otherTokens.size() << "\n";
-        }
-
-        TStringBuilder textBuilder;
-
-        for (size_t i = 0; i < Min(mainTokens.size(), otherTokens.size()); ++i) {
-            if (mainTokens[i].Name != otherTokens[i].Name || mainTokens[i].Content != otherTokens[i].Content) {
-                hasErrors = true;
-                Cerr << "Mismatch token #" << i << ", main: " << mainTokens[i].Name << ":" << mainTokens[i].Content
-                    << ", " << name << ": " << otherTokens[i].Name << ":" << otherTokens[i].Content << "\n";
-                Cerr << "Text sample: [";
-                TString text = textBuilder;
-                constexpr size_t LexerContextSample = 50;
-                Cerr << text.substr(text.size() >= LexerContextSample ? text.size() - LexerContextSample : 0u, LexerContextSample);
-                Cerr << "]\n";
-                break;
-            }
-
-            textBuilder << mainTokens[i].Content;
-        }
-    };
-
-    check("pure", pureTokens);
-    check("regex", regexTokens);
-    return !hasErrors;
+    return true;
 }
 
 class TStoreMappingFunctor: public NLastGetopt::IOptHandler {
diff --git a/yql/essentials/tools/sql2yql/ya.make b/yql/essentials/tools/sql2yql/ya.make
index f874b2d8830..81efdd0dc92 100644
--- a/yql/essentials/tools/sql2yql/ya.make
+++ b/yql/essentials/tools/sql2yql/ya.make
@@ -13,11 +13,9 @@ PEERDIR(
     yql/essentials/sql/v1
     yql/essentials/sql/pg
     yql/essentials/sql/v1/format
+    yql/essentials/sql/v1/lexer/check
     yql/essentials/sql/v1/lexer/antlr4
     yql/essentials/sql/v1/lexer/antlr4_ansi
-    yql/essentials/sql/v1/lexer/antlr4_pure
-    yql/essentials/sql/v1/lexer/antlr4_pure_ansi
-    yql/essentials/sql/v1/lexer/regex
     yql/essentials/sql/v1/proto_parser/antlr4
     yql/essentials/sql/v1/proto_parser/antlr4_ansi
 )
diff --git a/yql/essentials/tools/yql_facade_run/ya.make b/yql/essentials/tools/yql_facade_run/ya.make
index d08df659479..662276a8577 100644
--- a/yql/essentials/tools/yql_facade_run/ya.make
+++ b/yql/essentials/tools/yql_facade_run/ya.make
@@ -37,6 +37,7 @@ PEERDIR(
     yql/essentials/protos
     yql/essentials/sql/settings
     yql/essentials/sql/v1/format
+    yql/essentials/sql/v1/lexer/check
     yql/essentials/sql/v1/lexer/antlr4
     yql/essentials/sql/v1/lexer/antlr4_ansi
     yql/essentials/sql/v1/proto_parser/antlr4
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
index fdfe60a433e..64fef291635 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp
@@ -43,6 +43,7 @@
 #include <yql/essentials/sql/v1/format/sql_format.h>
 #include <yql/essentials/sql/v1/sql.h>
 #include <yql/essentials/sql/sql.h>
+#include <yql/essentials/sql/v1/lexer/check/check_lexers.h>
 #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
 #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
@@ -436,6 +437,7 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) {
     if (CustomTests) {
         opts.AddLongOption("test-antlr4", "Check antlr4 parser").NoArgument().SetFlag(&TestAntlr4);
         opts.AddLongOption("test-format", "Compare formatted query's AST with the original query's AST (only syntaxVersion=1 is supported)").NoArgument().SetFlag(&TestSqlFormat);
+        opts.AddLongOption("test-lexers", "Compare lexers").NoArgument().SetFlag(&TestLexers);
         opts.AddLongOption("validate-result-format", "Check that result-format can parse Result").NoArgument().SetFlag(&ValidateResultFormat);
     }
 
@@ -575,6 +577,24 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
         ctx.NextUniqueId = NPg::GetSqlLanguageParser()->GetContext().NextUniqueId;
     }
     IModuleResolver::TPtr moduleResolver;
+    TModuleResolver::TModuleChecker moduleChecker;
+    if (RunOptions_.TestLexers) {
+        moduleChecker = [](const TString& query, const TString& fileName, TExprContext& ctx) {
+            TIssues issues;
+            if (!NSQLTranslationV1::CheckLexers(TPosition(0, 0, fileName), query, issues)) {
+                auto issue = TIssue(TPosition(0, 0, fileName), "Lexers mismatched");
+                for (const auto& i : issues) {
+                    issue.AddSubIssue(MakeIntrusive<TIssue>(i));
+                }
+
+                ctx.AddError(issue);
+                return false;
+            }
+
+            return true;
+        };
+    }
+
     if (RunOptions_.MountConfig) {
         TModulesTable modules;
         FillUserDataTableFromFileSystem(*RunOptions_.MountConfig, RunOptions_.DataTable);
@@ -585,9 +605,11 @@ int TFacadeRunner::DoMain(int argc, const char *argv[]) {
             return -1;
         }
 
-        moduleResolver = std::make_shared<TModuleResolver>(translators, std::move(modules), ctx.NextUniqueId, ClusterMapping_, RunOptions_.SqlFlags, RunOptions_.Mode >= ERunMode::Validate);
+        moduleResolver = std::make_shared<TModuleResolver>(translators, std::move(modules), ctx.NextUniqueId,
+            ClusterMapping_, RunOptions_.SqlFlags, RunOptions_.Mode >= ERunMode::Validate, THolder<TExprContext>(), moduleChecker);
     } else {
-        if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, ClusterMapping_, RunOptions_.OptimizeLibs && RunOptions_.Mode >= ERunMode::Validate)) {
+        if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, ClusterMapping_,
+            RunOptions_.OptimizeLibs && RunOptions_.Mode >= ERunMode::Validate, moduleChecker)) {
             *RunOptions_.ErrStream << "Errors loading default YQL libraries:" << Endl;
             ctx.IssueManager.GetIssues().PrintTo(*RunOptions_.ErrStream);
             return -1;
@@ -746,6 +768,14 @@ int TFacadeRunner::DoRun(TProgramFactory& factory) {
                 return -1;
             }
         }
+        if (!fail && RunOptions_.TestLexers && 1 == RunOptions_.SyntaxVersion) {
+            TIssues issues;
+            if (!NSQLTranslationV1::CheckLexers({}, RunOptions_.ProgramText, issues)) {
+                *RunOptions_.ErrStream << "Lexers mismatched" << Endl;
+                issues.PrintTo(*RunOptions_.ErrStream);
+                return -1;
+            }
+        }
     } else {
         RunOptions_.PrintInfo("Parse YQL...");
         if (!program->ParseYql()) {
diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
index d6037766c6f..e7169322924 100644
--- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h
+++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h
@@ -86,6 +86,7 @@ public:
     bool TestAntlr4 = false;
     bool AssumeYdbOnClusterWithSlash = false;
     bool TestSqlFormat = false;
+    bool TestLexers = false;
     THashMap<TString, NSQLTranslation::TTableBindingSettings> Bindings;
 
     bool PrintAst = false;