YQL-19616 refactor test lexers from sql2yql, supported facade run tools

commit_hash:fb1727dda2b8c7d2ff42d4436c54cb7aa1ce4bc2

1 files changed, 4 insertions, 63 deletions

diff --git a/yql/essentials/tools/sql2yql/sql2yql.cpp b/yql/essentials/tools/sql2yql/sql2yql.cpp
index 8d185f72bb0..c3ffc588a13 100644
--- a/yql/essentials/tools/sql2yql/sql2yql.cpp
+++ b/yql/essentials/tools/sql2yql/sql2yql.cpp
@@ -6,11 +6,9 @@
 
 #include <yql/essentials/sql/sql.h>
 #include <yql/essentials/sql/v1/sql.h>
+#include <yql/essentials/sql/v1/lexer/check/check_lexers.h>
 #include <yql/essentials/sql/v1/lexer/antlr4/lexer.h>
 #include <yql/essentials/sql/v1/lexer/antlr4_ansi/lexer.h>
-#include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h>
-#include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h>
-#include <yql/essentials/sql/v1/lexer/regex/lexer.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4/proto_parser.h>
 #include <yql/essentials/sql/v1/proto_parser/antlr4_ansi/proto_parser.h>
 #include <yql/essentials/providers/common/provider/yql_provider_names.h>
@@ -155,71 +153,14 @@ bool TestFormat(
     return true;
 }
 
-bool TestLexers(
-    const TString& query
-) {
-    NSQLTranslationV1::TLexers lexers;
-    NSQLTranslation::TTranslationSettings settings;
+bool TestLexers(const TString& query) {
     NYql::TIssues issues;
-    if (!NSQLTranslation::ParseTranslationSettings(query, settings, issues)) {
-        Cerr << issues.ToString();
-        return false;
-    }
-
-    lexers.Antlr4 = NSQLTranslationV1::MakeAntlr4LexerFactory();
-    lexers.Antlr4Ansi = NSQLTranslationV1::MakeAntlr4AnsiLexerFactory();
-    lexers.Antlr4Pure = NSQLTranslationV1::MakeAntlr4PureLexerFactory();
-    lexers.Antlr4PureAnsi = NSQLTranslationV1::MakeAntlr4PureAnsiLexerFactory();
-    auto lexerMain = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Default);
-    auto lexerPure = NSQLTranslationV1::MakeLexer(lexers, settings.AnsiLexer, true, NSQLTranslationV1::ELexerFlavor::Pure);
-    auto lexerRegex = NSQLTranslationV1::MakeRegexLexerFactory(settings.AnsiLexer)->MakeLexer();
-    TVector<NSQLTranslation::TParsedToken> mainTokens;
-    if (!lexerMain->Tokenize(query, "", [&](auto token) { mainTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
-        Cerr << issues.ToString();
-        return false;
-    }
-
-    TVector<NSQLTranslation::TParsedToken> pureTokens;
-    if (!lexerPure->Tokenize(query, "", [&](auto token) { pureTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
+    if (!NSQLTranslationV1::CheckLexers({}, query, issues)) {
         Cerr << issues.ToString();
         return false;
     }
 
-    TVector<NSQLTranslation::TParsedToken> regexTokens;
-    if (!lexerRegex->Tokenize(query, "", [&](auto token) { regexTokens.push_back(token);}, issues, NSQLTranslation::SQL_MAX_PARSER_ERRORS)) {
-        Cerr << issues.ToString();
-        return false;
-    }
-
-    bool hasErrors = false;
-    auto check = [&](const char* name, const TVector<NSQLTranslation::TParsedToken>& otherTokens) {
-        if (mainTokens.size() != otherTokens.size()) {
-            hasErrors = true;
-            Cerr << "Mismatch token count, main: " << mainTokens.size() << ", " << name << ": " << otherTokens.size() << "\n";
-        }
-
-        TStringBuilder textBuilder;
-
-        for (size_t i = 0; i < Min(mainTokens.size(), otherTokens.size()); ++i) {
-            if (mainTokens[i].Name != otherTokens[i].Name || mainTokens[i].Content != otherTokens[i].Content) {
-                hasErrors = true;
-                Cerr << "Mismatch token #" << i << ", main: " << mainTokens[i].Name << ":" << mainTokens[i].Content
-                    << ", " << name << ": " << otherTokens[i].Name << ":" << otherTokens[i].Content << "\n";
-                Cerr << "Text sample: [";
-                TString text = textBuilder;
-                constexpr size_t LexerContextSample = 50;
-                Cerr << text.substr(text.size() >= LexerContextSample ? text.size() - LexerContextSample : 0u, LexerContextSample);
-                Cerr << "]\n";
-                break;
-            }
-
-            textBuilder << mainTokens[i].Content;
-        }
-    };
-
-    check("pure", pureTokens);
-    check("regex", regexTokens);
-    return !hasErrors;
+    return true;
 }
 
 class TStoreMappingFunctor: public NLastGetopt::IOptHandler {