YQL-19616 Fix regex lexer

Fixed regex lexer issues:

- `TSKIP` token recognition
- `HEXGIGITS` number recognition
- `EOF` token content

---

- Related to https://github.com/ydb-platform/ydb/issues/15129
- Related to https://github.com/vityaman/ydb/issues/11

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1190
commit_hash:497c39efcbbe4e387da523b5e2c8abaa6485d93b

3 files changed, 8 insertions, 5 deletions

diff --git a/yql/essentials/sql/v1/lexer/regex/lexer.cpp b/yql/essentials/sql/v1/lexer/regex/lexer.cpp
index b8ca033b0c6..9f96e444ac7 100644
--- a/yql/essentials/sql/v1/lexer/regex/lexer.cpp
+++ b/yql/essentials/sql/v1/lexer/regex/lexer.cpp
@@ -28,11 +28,14 @@ namespace NSQLTranslationV1 {
             : Grammar_(std::move(grammar))
             , Ansi_(ansi)
         {
+            RE2::Options custom;
+            custom.set_longest_match(true);
+
             for (const auto& [token, regex] : RegexByOtherName) {
                 if (token == CommentTokenName) {
                     CommentRegex_.Reset(new RE2(regex));
                 } else {
-                    OtherRegexes_.emplace_back(token, new RE2(regex));
+                    OtherRegexes_.emplace_back(token, new RE2(regex, custom));
                 }
             }
         }
@@ -62,7 +65,7 @@ namespace NSQLTranslationV1 {
                 onNextToken(std::move(matched));
             }
 
-            onNextToken(TParsedToken{.Name = "EOF"});
+            onNextToken(TParsedToken{.Name = "EOF", .Content = "<EOF>"});
             return errors == 0;
         }
 
@@ -110,7 +113,7 @@ namespace NSQLTranslationV1 {
             size_t count = 0;
             for (const auto& keyword : Grammar_.KeywordNames) {
                 const TStringBuf content = prefix.substr(0, keyword.length());
-                if (AsciiEqualsIgnoreCase(content, keyword)) {
+                if (AsciiEqualsIgnoreCase(content, NSQLReflect::TLexerGrammar::KeywordBlock(keyword))) {
                     matches.emplace_back(keyword, TString(content));
                     count += 1;
                 }
diff --git a/yql/essentials/sql/v1/lexer/regex/lexer_ut.cpp b/yql/essentials/sql/v1/lexer/regex/lexer_ut.cpp
index 03c84bcffe3..6ac25008b34 100644
--- a/yql/essentials/sql/v1/lexer/regex/lexer_ut.cpp
+++ b/yql/essentials/sql/v1/lexer/regex/lexer_ut.cpp
@@ -216,4 +216,4 @@ Y_UNIT_TEST_SUITE(RegexLexerTests) {
         Check("\" SELECT", "[INVALID] WS( ) SELECT EOF");
     }
 
-} // Y_UNIT_TEST_SUITE(RegexLexerTests)
\ No newline at end of file
+} // Y_UNIT_TEST_SUITE(RegexLexerTests)
diff --git a/yql/essentials/sql/v1/lexer/regex/regex_ut.cpp b/yql/essentials/sql/v1/lexer/regex/regex_ut.cpp
index dad0b2ebd2d..8f22bda5886 100644
--- a/yql/essentials/sql/v1/lexer/regex/regex_ut.cpp
+++ b/yql/essentials/sql/v1/lexer/regex/regex_ut.cpp
@@ -93,4 +93,4 @@ Y_UNIT_TEST_SUITE(SqlRegexTests) {
             Get(defaultRegexes, "COMMENT"));
     }
 
-} // Y_UNIT_TEST_SUITE(SqlRegexTests)
\ No newline at end of file
+} // Y_UNIT_TEST_SUITE(SqlRegexTests)