[YQL-14290] Refactor ILexer interface

ref:8afd2a32e4f9fc5d449eb805f87349953471adb4

5 files changed, 26 insertions, 19 deletions

diff --git a/ydb/library/yql/parser/lexer_common/lexer.h b/ydb/library/yql/parser/lexer_common/lexer.h
index ad19f09b8f..bc59c37e56 100644
--- a/ydb/library/yql/parser/lexer_common/lexer.h
+++ b/ydb/library/yql/parser/lexer_common/lexer.h
@@ -4,6 +4,8 @@
 #include <util/generic/vector.h>
 #include <util/stream/output.h>
 
+#include <functional>
+
 namespace NYql {
 
 class TIssues;
@@ -21,17 +23,19 @@ struct TParsedToken {
     ui32 LinePos = 0; // starts from 0
 };
 
-using TParsedTokenList = TVector<TParsedToken>;
-
-IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end);
-
 class ILexer {
 public:
     using TPtr = THolder<ILexer>;
+    using TTokenCallback = std::function<void(TParsedToken&& token)>;
 
-    virtual bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) = 0;
+    virtual bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) = 0;
     virtual ~ILexer() = default;
 };
 
+using TParsedTokenList = TVector<TParsedToken>;
+
+IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator begin, TParsedTokenList::const_iterator end);
+bool Tokenize(const ILexer::TPtr& lexer, const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors);
+
 }
 
diff --git a/ydb/library/yql/parser/lexer_common/tokens.cpp b/ydb/library/yql/parser/lexer_common/tokens.cpp
index 87a5dd3088..db3553b2ab 100644
--- a/ydb/library/yql/parser/lexer_common/tokens.cpp
+++ b/ydb/library/yql/parser/lexer_common/tokens.cpp
@@ -10,4 +10,13 @@ IOutputStream& OutputTokens(IOutputStream& out, TParsedTokenList::const_iterator
     return out;
 }
 
+bool Tokenize(const ILexer::TPtr& lexer, const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) {
+    auto onNextToken = [&tokens](TParsedToken&& token) {
+        tokens.push_back(std::move(token));
+    };
+
+    return lexer->Tokenize(query, queryName, onNextToken, issues, maxErrors);
+}
+
+
 }
diff --git a/ydb/library/yql/parser/proto_ast/proto_ast.h b/ydb/library/yql/parser/proto_ast/proto_ast.h
index 5bb8a9f5f9..6bc8db23a9 100644
--- a/ydb/library/yql/parser/proto_ast/proto_ast.h
+++ b/ydb/library/yql/parser/proto_ast/proto_ast.h
@@ -114,8 +114,7 @@ namespace NProtoAST {
         {
         }
 
-        NSQLTranslation::TParsedTokenList CollectTokens(IErrorCollector& errors) {
-            NSQLTranslation::TParsedTokenList result;
+        void CollectTokens(IErrorCollector& errors, const NSQLTranslation::ILexer::TTokenCallback& onNextToken) {
             try {
                 Lexer.ReportErrors(&errors);
                 auto src = Lexer.get_tokSource();
@@ -123,22 +122,19 @@ namespace NProtoAST {
                     auto token = src->nextToken();
                     auto type = token->getType();
                     const bool isEOF = type == TLexer::CommonTokenType::TOKEN_EOF;
-                    result.emplace_back();
-                    NSQLTranslation::TParsedToken& last = result.back();
+                    NSQLTranslation::TParsedToken last;
                     last.Name = isEOF ? "EOF" : TokenNames[type];
                     last.Content = token->getText();
                     last.Line = token->get_line();
                     last.LinePos = token->get_charPositionInLine();
+                    onNextToken(std::move(last));
                     if (isEOF) {
                         break;
                     }
                 }
-                return result;
             } catch (const TTooManyErrors&) {
-                return result;
             } catch (const yexception& e) {
                 errors.Error(0, 0, e.what());
-                return result;
             }
         }
 
diff --git a/ydb/library/yql/sql/v0/lexer/lexer.cpp b/ydb/library/yql/sql/v0/lexer/lexer.cpp
index 698a1d4257..b28fb5536d 100644
--- a/ydb/library/yql/sql/v0/lexer/lexer.cpp
+++ b/ydb/library/yql/sql/v0/lexer/lexer.cpp
@@ -21,20 +21,19 @@ TMutex SanitizerSQLTranslationMutex;
 #endif
 
 using NSQLTranslation::ILexer;
-using NSQLTranslation::TParsedTokenList;
 
 class TV0Lexer : public ILexer {
 public:
     TV0Lexer() = default;
 
-    bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) override {
+    bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) override {
         issues.Clear();
 #if defined(_tsan_enabled_)
         TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
 #endif
         NSQLTranslation::TErrorCollectorOverIssues collector(issues, maxErrors, "");
         NProtoAST::TLexerTokensCollector<NALP::SQLLexer> tokensCollector(query, (const char**)NALP::SQLParserTokenNames, queryName);
-        tokens = tokensCollector.CollectTokens(collector);
+        tokensCollector.CollectTokens(collector, onNextToken);
         return !AnyOf(issues.begin(), issues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });
     }
 };
diff --git a/ydb/library/yql/sql/v1/lexer/lexer.cpp b/ydb/library/yql/sql/v1/lexer/lexer.cpp
index 3f58f9d236..81b58e3db9 100644
--- a/ydb/library/yql/sql/v1/lexer/lexer.cpp
+++ b/ydb/library/yql/sql/v1/lexer/lexer.cpp
@@ -26,7 +26,6 @@ TMutex SanitizerSQLTranslationMutex;
 #endif
 
 using NSQLTranslation::ILexer;
-using NSQLTranslation::TParsedTokenList;
 
 class TV1Lexer : public ILexer {
 public:
@@ -35,7 +34,7 @@ public:
     {
     }
 
-    bool Tokenize(const TString& query, const TString& queryName, TParsedTokenList& tokens, NYql::TIssues& issues, size_t maxErrors) override {
+    bool Tokenize(const TString& query, const TString& queryName, const TTokenCallback& onNextToken, NYql::TIssues& issues, size_t maxErrors) override {
         issues.Clear();
 #if defined(_tsan_enabled_)
         TGuard<TMutex> grd(SanitizerSQLTranslationMutex);
@@ -43,10 +42,10 @@ public:
         NSQLTranslation::TErrorCollectorOverIssues collector(issues, maxErrors, "");
         if (Ansi) {
             NProtoAST::TLexerTokensCollector<NALPAnsi::SQLv1Lexer> tokensCollector(query, (const char**)NALPAnsi::SQLv1ParserTokenNames, queryName);
-            tokens = tokensCollector.CollectTokens(collector);
+            tokensCollector.CollectTokens(collector, onNextToken);
         } else {
             NProtoAST::TLexerTokensCollector<NALPDefault::SQLv1Lexer> tokensCollector(query, (const char**)NALPDefault::SQLv1ParserTokenNames, queryName);
-            tokens = tokensCollector.CollectTokens(collector);
+            tokensCollector.CollectTokens(collector, onNextToken);
         }
 
         return !AnyOf(issues.begin(), issues.end(), [](auto issue) { return issue.GetSeverity() == NYql::ESeverity::TSeverityIds_ESeverityId_S_ERROR; });