YQL-19747 Normalize names for ranking and filtering

I was lazy to search for a most frequent used name among equivalent by the relation `(a ~ b) iff (NormalizeName(a) = NormalizeName(b))`. Because it seems that names we receive from JSONs are canonized and therefore in a preferable style by the opinion of the YQL language designers. But because of duplicates at `statements_opensource.json` we have, for example, both `IGNORETYPEV3` and `IGNORE_TYPE_V3` in candidates list. I think that we should just remove `IGNORETYPEV3` from the JSON.

---

- Related to https://github.com/ydb-platform/ydb/issues/9056
- Related to https://github.com/vityaman/ydb/issues/21

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1229
commit_hash:fe73374ae27df1fcacb0adccda930ec98ed1d7a6

1 files changed, 12 insertions, 2 deletions

diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
index a0681b1888f..d14d7b85442 100644
--- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
@@ -605,7 +605,6 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
         {
             TVector<TCandidate> expected = {
                 {HintName, "IGNORE_TYPE_V3"},
-                {HintName, "IGNORETYPEV3"},
             };
             UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"REDUCE a WITH ig"}), expected);
         }
@@ -642,6 +641,17 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
         UNIT_ASSERT_GE(Complete(engine, {"SELECT "}).size(), 55);
     }
 
+    Y_UNIT_TEST(NameNormalization) {
+        auto set = MakeDefaultNameSet();
+        auto service = MakeStaticNameService(std::move(set), MakeDefaultRanking());
+        auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
+
+        TVector<TCandidate> expected = {
+            {HintName, "IGNORE_TYPE_V3"},
+        };
+        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"REDUCE a WITH ignoret"}), expected);
+    }
+
     Y_UNIT_TEST(Ranking) {
         TFrequencyData frequency = {
             .Keywords = {
@@ -715,7 +725,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
                 {HintName, "XLOCK"},
                 {HintName, "UNORDERED"},
                 {Keyword, "COLUMNS"},
-                {HintName, "FORCEINFERSCHEMA"},
+                {HintName, "FORCE_INFER_SCHEMA"},
             };
             UNIT_ASSERT_VALUES_EQUAL(CompleteTop(expected.size(), engine, {"SELECT * FROM a WITH "}), expected);
         }