YQL-19616 Convert YQL lexer grammar to regexes

- [x] Parse YQL grammar to extract lexer grammar into `TLexerGrammar`.
- [x] Translate `TLexerGrammar` into regexes.
- [x] Implement a lexer via regexes `TRegexLexer` to test generated regexes validity.
- [x] Test on `Default` syntax mode.
- [x] Test on `ANSI` syntax mode.

---
- Related to https://github.com/ydb-platform/ydb/issues/15129
- Requirement for https://github.com/ytsaurus/ytsaurus/pull/1112

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1127
commit_hash:03ffffe81cdafe7f93a4d3fd9a3212fe67f1c72d

1 files changed, 46 insertions, 0 deletions

diff --git a/yql/essentials/sql/v1/reflect/sql_reflect_ut.cpp b/yql/essentials/sql/v1/reflect/sql_reflect_ut.cpp
new file mode 100644
index 00000000000..7bef2879e55
--- /dev/null
+++ b/yql/essentials/sql/v1/reflect/sql_reflect_ut.cpp
@@ -0,0 +1,46 @@
+#include "sql_reflect.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+using namespace NSQLReflect;
+
+namespace {
+    TLexerGrammar grammar = LoadLexerGrammar();
+} // namespace
+
+Y_UNIT_TEST_SUITE(SqlReflectTests) {
+    Y_UNIT_TEST(Keywords) {
+        UNIT_ASSERT_VALUES_EQUAL(grammar.KeywordNames.contains("SELECT"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.KeywordNames.contains("INSERT"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.KeywordNames.contains("WHERE"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.KeywordNames.contains("COMMIT"), true);
+    }
+
+    Y_UNIT_TEST(Punctuation) {
+        UNIT_ASSERT_VALUES_EQUAL(grammar.PunctuationNames.contains("LPAREN"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.BlockByName.at("LPAREN"), "(");
+
+        UNIT_ASSERT_VALUES_EQUAL(grammar.PunctuationNames.contains("MINUS"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.BlockByName.at("MINUS"), "-");
+
+        UNIT_ASSERT_VALUES_EQUAL(grammar.PunctuationNames.contains("NAMESPACE"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.BlockByName.at("NAMESPACE"), "::");
+    }
+
+    Y_UNIT_TEST(Other) {
+        UNIT_ASSERT_VALUES_EQUAL(grammar.OtherNames.contains("REAL"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.OtherNames.contains("STRING_VALUE"), true);
+        UNIT_ASSERT_VALUES_EQUAL(grammar.OtherNames.contains("STRING_MULTILINE"), false);
+
+        UNIT_ASSERT_VALUES_EQUAL(
+            grammar.BlockByName.at("FLOAT_EXP"),
+            "E (PLUS | MINUS)? DECDIGITS");
+        UNIT_ASSERT_VALUES_EQUAL(
+            grammar.BlockByName.at("STRING_MULTILINE"),
+            "(DOUBLE_COMMAT .*? DOUBLE_COMMAT)+ COMMAT?");
+        UNIT_ASSERT_VALUES_EQUAL(
+            grammar.BlockByName.at("REAL"),
+            "(DECDIGITS DOT DIGIT* FLOAT_EXP? | DECDIGITS FLOAT_EXP) (F | P (F ('4' | '8') | N)?)?");
+    }
+
+} // Y_UNIT_TEST_SUITE(SqlReflectTests)