YQL-19747 Enable custom NameSet and FrequencyData

Clients might want to use custom `NameSet` and `FrequencyData` in their environment, for example, to get their private UDFs and to have more relevant ranking that includes their private UDF and respect their usage pattern.

To achieve this goal I decided to load pure `NameSet` and `FrequencyData` and provide functions for pruning. Also I checked defaults and decided that it is more common for a client to create a `StaticNameService` from pure `NameSet` and `FrequencyData` to keep their pruning consistent.

I also extracted a separate module `ranking`. It will be needed when I will implement `UnionNameService` to union `StaticNameService` and `SchemaNameService`. `UnionNameService` will load `Limit` names from each child and then crop them to sorted prefix of length `Limit` using `IRanking`.

---

- Related to `YQL-19747`
- Related to https://github.com/ydb-platform/ydb/issues/9056
- Related to https://github.com/vityaman/ydb/issues/14

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1246

Co-authored-by: vvvv <[email protected]>
Co-authored-by: vvvv <[email protected]>
commit_hash:cdca301a58a34e56d537a447b4ff779cd70faea6

17 files changed, 196 insertions, 140 deletions

diff --git a/yql/essentials/sql/v1/complete/name/service/static/frequency.cpp b/yql/essentials/sql/v1/complete/name/service/ranking/frequency.cpp
index 456a9feebcd..a9a99e27097 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/frequency.cpp
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/frequency.cpp
@@ -1,6 +1,6 @@
 #include "frequency.h"
 
-#include "name_index.h"
+#include <yql/essentials/core/sql_types/normalize_name.h>
 
 #include <library/cpp/json/json_reader.h>
 #include <library/cpp/resource/resource.h>
@@ -56,20 +56,9 @@ namespace NSQLComplete {
         }
     };
 
-    TFrequencyData Convert(TVector<TFrequencyItem> items, auto normalize) {
+    TFrequencyData Collect(const TVector<TFrequencyItem>& items) {
         TFrequencyData data;
         for (auto& item : items) {
-            if (item.Parent == Json.Parent.Pragma ||
-                item.Parent == Json.Parent.Type ||
-                item.Parent == Json.Parent.Func ||
-                item.Parent == Json.Parent.Keyword ||
-                item.Parent == Json.Parent.ModuleFunc ||
-                item.Parent == Json.Parent.Module ||
-                item.Parent == Json.Parent.ReadHint ||
-                item.Parent == Json.Parent.InsertHint) {
-                item.Rule = normalize(item.Rule);
-            }
-
             if (item.Parent == Json.Parent.Pragma) {
                 data.Pragmas[item.Rule] += item.Sum;
             } else if (item.Parent == Json.Parent.Type) {
@@ -91,24 +80,38 @@ namespace NSQLComplete {
         return data;
     }
 
-    TFrequencyData ParseJsonFrequencyData(const TStringBuf text, auto normalize) {
-        return Convert(TFrequencyItem::ParseListFromJsonText(text), normalize);
+    THashMap<TString, size_t> PrunedBy(const THashMap<TString, size_t>& data, auto normalize) {
+        THashMap<TString, size_t> pruned;
+        for (const auto& [name, count] : data) {
+            pruned[normalize(name)] += count;
+        }
+        return pruned;
     }
 
-    TFrequencyData ParseJsonFrequencyData(const TStringBuf text) {
-        return ParseJsonFrequencyData(text, NormalizeName);
+    TFrequencyData PrunedBy(const TFrequencyData& data, auto normalize) {
+        return {
+            .Keywords = PrunedBy(data.Keywords, normalize),
+            .Pragmas = PrunedBy(data.Pragmas, normalize),
+            .Types = PrunedBy(data.Types, normalize),
+            .Functions = PrunedBy(data.Functions, normalize),
+            .Hints = PrunedBy(data.Hints, normalize),
+        };
     }
 
-    TFrequencyData LoadFrequencyData() {
-        TString text;
-        Y_ENSURE(NResource::FindExact("rules_corr_basic.json", &text));
-        return ParseJsonFrequencyData(text, NormalizeName);
+    TFrequencyData Pruned(const TFrequencyData& data) {
+        return PrunedBy(data, [](TStringBuf s) {
+            return NYql::NormalizeName(s);
+        });
     }
 
-    TFrequencyData LoadFrequencyDataForPrunning() {
+    TFrequencyData ParseJsonFrequencyData(const TStringBuf text) {
+        return Collect(TFrequencyItem::ParseListFromJsonText(text));
+    }
+
+    TFrequencyData LoadFrequencyData() {
         TString text;
         Y_ENSURE(NResource::FindExact("rules_corr_basic.json", &text));
-        return ParseJsonFrequencyData(text, UnchangedName);
+        return ParseJsonFrequencyData(text);
     }
 
 } // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/static/frequency.h b/yql/essentials/sql/v1/complete/name/service/ranking/frequency.h
index 0a5dc36dfbd..1e729bac1d3 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/frequency.h
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/frequency.h
@@ -13,10 +13,10 @@ namespace NSQLComplete {
         THashMap<TString, size_t> Hints;
     };
 
+    TFrequencyData Pruned(const TFrequencyData& data);
+
     TFrequencyData ParseJsonFrequencyData(const TStringBuf text);
 
     TFrequencyData LoadFrequencyData();
 
-    TFrequencyData LoadFrequencyDataForPrunning();
-
 } // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/static/frequency_ut.cpp b/yql/essentials/sql/v1/complete/name/service/ranking/frequency_ut.cpp
index c630f0ca987..725abfc562d 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/frequency_ut.cpp
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/frequency_ut.cpp
@@ -7,7 +7,7 @@ using namespace NSQLComplete;
 Y_UNIT_TEST_SUITE(FrequencyTests) {
 
     Y_UNIT_TEST(FrequencyDataJson) {
-        TFrequencyData actual = ParseJsonFrequencyData(R"([
+        TFrequencyData actual = Pruned(ParseJsonFrequencyData(R"([
             {"parent":"FUNC","rule":"ABC","sum":1},
             {"parent":"TYPE","rule":"BIGINT","sum":7101},
             {"parent":"KEYWORD","rule":"UNION","sum":65064443},
@@ -16,7 +16,7 @@ Y_UNIT_TEST_SUITE(FrequencyTests) {
             {"parent":"READ_HINT","rule":"COLUMNS","sum":826110},
             {"parent":"INSERT_HINT","rule":"COLUMN_GROUPS","sum":225},
             {"parent":"TRule_action_or_subquery_args","rule":"TRule_action_or_subquery_args.Block2","sum":4874480}
-        ])");
+        ])"));
 
         TFrequencyData expected = {
             .Keywords = {
diff --git a/yql/essentials/sql/v1/complete/name/service/static/ranking.cpp b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp
index 6ef3817defb..901b841d1b5 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/ranking.cpp
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp
@@ -1,7 +1,5 @@
 #include "ranking.h"
 
-#include "frequency.h"
-
 #include <yql/essentials/sql/v1/complete/name/service/name_service.h>
 
 #include <yql/essentials/core/sql_types/normalize_name.h>
@@ -18,8 +16,8 @@ namespace NSQLComplete {
         };
 
     public:
-        TRanking(TFrequencyData frequency)
-            : Frequency_(std::move(frequency))
+        explicit TRanking(TFrequencyData frequency)
+            : Frequency_(frequency)
         {
         }
 
@@ -115,11 +113,11 @@ namespace NSQLComplete {
     };
 
     IRanking::TPtr MakeDefaultRanking() {
-        return MakeIntrusive<TRanking>(LoadFrequencyData());
+        return MakeDefaultRanking(LoadFrequencyData());
     }
 
-    IRanking::TPtr MakeDefaultRanking(TFrequencyData frequency) {
-        return MakeIntrusive<TRanking>(frequency);
+    IRanking::TPtr MakeDefaultRanking(const TFrequencyData& frequency) {
+        return MakeIntrusive<TRanking>(Pruned(frequency));
     }
 
 } // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.h b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.h
new file mode 100644
index 00000000000..269f46d2028
--- /dev/null
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h>
+#include <yql/essentials/sql/v1/complete/name/service/name_service.h>
+
+#include <util/generic/hash.h>
+
+namespace NSQLComplete {
+
+    class IRanking: public TThrRefBase {
+    public:
+        using TPtr = TIntrusivePtr<IRanking>;
+
+        virtual void CropToSortedPrefix(TVector<TGenericName>& names, size_t limit) const = 0;
+        virtual ~IRanking() = default;
+    };
+
+    // TODO(YQL-19747): Migrate YDB CLI to MakeDefaultRanking(...)
+    IRanking::TPtr MakeDefaultRanking();
+
+    IRanking::TPtr MakeDefaultRanking(const TFrequencyData& frequency);
+
+} // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/ranking/ut/ya.make b/yql/essentials/sql/v1/complete/name/service/ranking/ut/ya.make
new file mode 100644
index 00000000000..f0c5333535f
--- /dev/null
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/ut/ya.make
@@ -0,0 +1,7 @@
+UNITTEST_FOR(yql/essentials/sql/v1/complete/name/service/ranking)
+
+SRCS(
+    frequency_ut.cpp
+)
+
+END()
diff --git a/yql/essentials/sql/v1/complete/name/service/ranking/ya.make b/yql/essentials/sql/v1/complete/name/service/ranking/ya.make
new file mode 100644
index 00000000000..4b139666fea
--- /dev/null
+++ b/yql/essentials/sql/v1/complete/name/service/ranking/ya.make
@@ -0,0 +1,21 @@
+LIBRARY()
+
+SRCS(
+    frequency.cpp
+    ranking.cpp
+)
+
+PEERDIR(
+    yql/essentials/core/sql_types
+    yql/essentials/sql/v1/complete/name/service
+)
+
+RESOURCE(
+    yql/essentials/data/language/rules_corr_basic.json rules_corr_basic.json
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+    ut
+)
diff --git a/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp
index 7b3c3f1bd1a..bb8d0840a93 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp
+++ b/yql/essentials/sql/v1/complete/name/service/static/name_service.cpp
@@ -1,8 +1,8 @@
 #include "name_service.h"
 
 #include "name_index.h"
-#include "ranking.h"
 
+#include <yql/essentials/sql/v1/complete/name/service/ranking/ranking.h>
 #include <yql/essentials/sql/v1/complete/text/case.h>
 
 namespace NSQLComplete {
@@ -72,7 +72,7 @@ namespace NSQLComplete {
 
     class TStaticNameService: public INameService {
     public:
-        explicit TStaticNameService(NameSet names, IRanking::TPtr ranking)
+        explicit TStaticNameService(TNameSet names, IRanking::TPtr ranking)
             : Pragmas_(BuildNameIndex(std::move(names.Pragmas), NormalizeName))
             , Types_(BuildNameIndex(std::move(names.Types), NormalizeName))
             , Functions_(BuildNameIndex(std::move(names.Functions), NormalizeName))
@@ -139,11 +139,13 @@ namespace NSQLComplete {
         IRanking::TPtr Ranking_;
     };
 
-    INameService::TPtr MakeStaticNameService() {
-        return MakeStaticNameService(MakeDefaultNameSet(), MakeDefaultRanking());
+    INameService::TPtr MakeStaticNameService(TNameSet names, TFrequencyData frequency) {
+        return INameService::TPtr(new TStaticNameService(
+            Pruned(std::move(names), frequency),
+            MakeDefaultRanking(std::move(frequency))));
     }
 
-    INameService::TPtr MakeStaticNameService(NameSet names, IRanking::TPtr ranking) {
+    INameService::TPtr MakeStaticNameService(TNameSet names, IRanking::TPtr ranking) {
         return MakeIntrusive<TStaticNameService>(std::move(names), std::move(ranking));
     }
 
diff --git a/yql/essentials/sql/v1/complete/name/service/static/name_service.h b/yql/essentials/sql/v1/complete/name/service/static/name_service.h
index b0ca2b2ff81..531d3882ccc 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/name_service.h
+++ b/yql/essentials/sql/v1/complete/name/service/static/name_service.h
@@ -1,22 +1,16 @@
 #pragma once
 
-#include "ranking.h"
+#include "name_set.h"
 
+#include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h>
+#include <yql/essentials/sql/v1/complete/name/service/ranking/ranking.h>
 #include <yql/essentials/sql/v1/complete/name/service/name_service.h>
 
 namespace NSQLComplete {
 
-    struct NameSet {
-        TVector<TString> Pragmas;
-        TVector<TString> Types;
-        TVector<TString> Functions;
-        THashMap<EStatementKind, TVector<TString>> Hints;
-    };
+    INameService::TPtr MakeStaticNameService(TNameSet names, TFrequencyData frequency);
 
-    NameSet MakeDefaultNameSet();
-
-    INameService::TPtr MakeStaticNameService();
-
-    INameService::TPtr MakeStaticNameService(NameSet names, IRanking::TPtr ranking);
+    // TODO(YQL-19747): Migrate YDB CLI to method from pure TNameSet and TFrequencyData
+    INameService::TPtr MakeStaticNameService(TNameSet names, IRanking::TPtr ranking);
 
 } // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/static/name_set.cpp b/yql/essentials/sql/v1/complete/name/service/static/name_set.cpp
new file mode 100644
index 00000000000..eb01c78420c
--- /dev/null
+++ b/yql/essentials/sql/v1/complete/name/service/static/name_set.cpp
@@ -0,0 +1,43 @@
+#include "name_set.h"
+
+#include "name_index.h"
+
+namespace NSQLComplete {
+
+    TVector<TString> Pruned(TVector<TString> names, const THashMap<TString, size_t>& frequency) {
+        THashMap<TString, TVector<std::tuple<TString, size_t>>> groups;
+
+        for (auto& [normalized, original] : BuildNameIndex(std::move(names), NormalizeName)) {
+            size_t freq = 0;
+            if (const size_t* it = frequency.FindPtr(original)) {
+                freq = *it;
+            }
+            groups[normalized].emplace_back(std::move(original), freq);
+        }
+
+        for (auto& [_, group] : groups) {
+            Sort(group, [](const auto& lhs, const auto& rhs) {
+                return std::get<1>(lhs) < std::get<1>(rhs);
+            });
+        }
+
+        names = TVector<TString>();
+        names.reserve(groups.size());
+        for (auto& [_, group] : groups) {
+            Y_ASSERT(!group.empty());
+            names.emplace_back(std::move(std::get<0>(group.back())));
+        }
+        return names;
+    }
+
+    TNameSet Pruned(TNameSet names, const TFrequencyData& frequency) {
+        names.Pragmas = Pruned(std::move(names.Pragmas), frequency.Pragmas);
+        names.Types = Pruned(std::move(names.Types), frequency.Types);
+        names.Functions = Pruned(std::move(names.Functions), frequency.Functions);
+        for (auto& [k, h] : names.Hints) {
+            h = Pruned(h, frequency.Hints);
+        }
+        return names;
+    }
+
+} // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/static/name_set.h b/yql/essentials/sql/v1/complete/name/service/static/name_set.h
new file mode 100644
index 00000000000..fd5d01b69d4
--- /dev/null
+++ b/yql/essentials/sql/v1/complete/name/service/static/name_set.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <yql/essentials/sql/v1/complete/core/statement.h>
+#include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h>
+
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+#include <util/generic/hash.h>
+
+namespace NSQLComplete {
+
+    struct TNameSet {
+        TVector<TString> Pragmas;
+        TVector<TString> Types;
+        TVector<TString> Functions;
+        THashMap<EStatementKind, TVector<TString>> Hints;
+    };
+
+    // TODO(YQL-19747): Migrate YDB CLI to TNameSet
+    using NameSet = TNameSet;
+
+    TNameSet Pruned(TNameSet names, const TFrequencyData& frequency);
+
+    TNameSet LoadDefaultNameSet();
+
+    // TODO(YQL-19747): Mirate YDB CLI to LoadDefaultNameSet
+    TNameSet MakeDefaultNameSet();
+
+} // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/static/json_name_set.cpp b/yql/essentials/sql/v1/complete/name/service/static/name_set_json.cpp
index a30f62aac20..fc802b1d886 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/json_name_set.cpp
+++ b/yql/essentials/sql/v1/complete/name/service/static/name_set_json.cpp
@@ -1,7 +1,6 @@
-#include "name_service.h"
+#include "name_set.h"
 
-#include "frequency.h"
-#include "name_index.h"
+#include <yql/essentials/sql/v1/complete/name/service/name_service.h>
 
 #include <library/cpp/json/json_reader.h>
 #include <library/cpp/resource/resource.h>
@@ -81,52 +80,19 @@ namespace NSQLComplete {
         return hints;
     }
 
-    TVector<TString> Pruned(TVector<TString> names, const THashMap<TString, size_t>& frequency) {
-        THashMap<TString, TVector<std::tuple<TString, size_t>>> groups;
-
-        for (auto& [normalized, original] : BuildNameIndex(std::move(names), NormalizeName)) {
-            size_t freq = 0;
-            if (const size_t* it = frequency.FindPtr(original)) {
-                freq = *it;
-            }
-            groups[normalized].emplace_back(std::move(original), freq);
-        }
-
-        for (auto& [_, group] : groups) {
-            Sort(group, [](const auto& lhs, const auto& rhs) {
-                return std::get<1>(lhs) < std::get<1>(rhs);
-            });
-        }
-
-        names = TVector<TString>();
-        names.reserve(groups.size());
-        for (auto& [_, group] : groups) {
-            Y_ASSERT(!group.empty());
-            names.emplace_back(std::move(std::get<0>(group.back())));
-        }
-        return names;
-    }
-
-    NameSet Pruned(NameSet names) {
-        auto frequency = LoadFrequencyDataForPrunning();
-        names.Pragmas = Pruned(std::move(names.Pragmas), frequency.Pragmas);
-        names.Types = Pruned(std::move(names.Types), frequency.Types);
-        names.Functions = Pruned(std::move(names.Functions), frequency.Functions);
-        for (auto& [k, h] : names.Hints) {
-            h = Pruned(h, frequency.Hints);
-        }
-        return names;
-    }
-
-    NameSet MakeDefaultNameSet() {
-        return Pruned({
+    TNameSet LoadDefaultNameSet() {
+        return {
             .Pragmas = ParsePragmas(LoadJsonResource("pragmas_opensource.json")),
             .Types = ParseTypes(LoadJsonResource("types.json")),
             .Functions = Merge(
                 ParseFunctions(LoadJsonResource("sql_functions.json")),
                 ParseUdfs(LoadJsonResource("udfs_basic.json"))),
             .Hints = ParseHints(LoadJsonResource("statements_opensource.json")),
-        });
+        };
+    }
+
+    TNameSet MakeDefaultNameSet() {
+        return Pruned(LoadDefaultNameSet(), LoadFrequencyData());
     }
 
 } // namespace NSQLComplete
diff --git a/yql/essentials/sql/v1/complete/name/service/static/ranking.h b/yql/essentials/sql/v1/complete/name/service/static/ranking.h
index bc1f57f9a51..947f33065cd 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/ranking.h
+++ b/yql/essentials/sql/v1/complete/name/service/static/ranking.h
@@ -1,23 +1,4 @@
 #pragma once
 
-#include "frequency.h"
-
-#include <yql/essentials/sql/v1/complete/name/service/name_service.h>
-
-#include <util/generic/hash.h>
-
-namespace NSQLComplete {
-
-    class IRanking: public TThrRefBase {
-    public:
-        using TPtr = TIntrusivePtr<IRanking>;
-
-        virtual void CropToSortedPrefix(TVector<TGenericName>& names, size_t limit) const = 0;
-        virtual ~IRanking() = default;
-    };
-
-    IRanking::TPtr MakeDefaultRanking();
-
-    IRanking::TPtr MakeDefaultRanking(TFrequencyData frequency);
-
-} // namespace NSQLComplete
+// TODO(YQL-19747): Migrate YDB CLI
+#include <yql/essentials/sql/v1/complete/name/service/ranking/ranking.h>
diff --git a/yql/essentials/sql/v1/complete/name/service/static/ut/ya.make b/yql/essentials/sql/v1/complete/name/service/static/ut/ya.make
deleted file mode 100644
index a486db3cd0d..00000000000
--- a/yql/essentials/sql/v1/complete/name/service/static/ut/ya.make
+++ /dev/null
@@ -1,7 +0,0 @@
-UNITTEST_FOR(yql/essentials/sql/v1/complete/name/service/static)
-
-SRCS(
-    frequency_ut.cpp
-)
-
-END()
diff --git a/yql/essentials/sql/v1/complete/name/service/static/ya.make b/yql/essentials/sql/v1/complete/name/service/static/ya.make
index 95417316688..303916474ba 100644
--- a/yql/essentials/sql/v1/complete/name/service/static/ya.make
+++ b/yql/essentials/sql/v1/complete/name/service/static/ya.make
@@ -1,16 +1,16 @@
 LIBRARY()
 
 SRCS(
-    frequency.cpp
-    json_name_set.cpp
+    name_set_json.cpp
+    name_set.cpp
     name_index.cpp
     name_service.cpp
-    ranking.cpp
 )
 
 PEERDIR(
     yql/essentials/core/sql_types
     yql/essentials/sql/v1/complete/name/service
+    yql/essentials/sql/v1/complete/name/service/ranking
     yql/essentials/sql/v1/complete/text
 )
 
@@ -24,7 +24,3 @@ RESOURCE(
 )
 
 END()
-
-RECURSE_FOR_TESTS(
-    ut
-)
diff --git a/yql/essentials/sql/v1/complete/name/service/ya.make b/yql/essentials/sql/v1/complete/name/service/ya.make
index 7ab6f698b17..473ee05f7d8 100644
--- a/yql/essentials/sql/v1/complete/name/service/ya.make
+++ b/yql/essentials/sql/v1/complete/name/service/ya.make
@@ -7,5 +7,6 @@ PEERDIR(
 END()
 
 RECURSE(
+    ranking
     static
 )
diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
index b614f561492..54bcc35233f 100644
--- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
@@ -1,8 +1,8 @@
 #include "sql_complete.h"
 
-#include <yql/essentials/sql/v1/complete/name/service/static/frequency.h>
+#include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h>
+#include <yql/essentials/sql/v1/complete/name/service/ranking/ranking.h>
 #include <yql/essentials/sql/v1/complete/name/service/static/name_service.h>
-#include <yql/essentials/sql/v1/complete/name/service/static/ranking.h>
 
 #include <yql/essentials/sql/v1/lexer/lexer.h>
 #include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h>
@@ -49,7 +49,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
 
     ISqlCompletionEngine::TPtr MakeSqlCompletionEngineUT() {
         TLexerSupplier lexer = MakePureLexerSupplier();
-        NameSet names = {
+        TNameSet names = {
             .Pragmas = {"yson.CastToString"},
             .Types = {"Uint64"},
             .Functions = {"StartsWith", "DateTime::Split"},
@@ -58,8 +58,8 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
                 {EStatementKind::Insert, {"EXPIRATION"}},
             },
         };
-        auto ranking = MakeDefaultRanking({});
-        INameService::TPtr service = MakeStaticNameService(std::move(names), std::move(ranking));
+        TFrequencyData frequency = {};
+        INameService::TPtr service = MakeStaticNameService(std::move(names), std::move(frequency));
         return MakeSqlCompletionEngine(std::move(lexer), std::move(service));
     }
 
@@ -669,8 +669,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
     }
 
     Y_UNIT_TEST(DefaultNameService) {
-        auto set = MakeDefaultNameSet();
-        auto service = MakeStaticNameService(std::move(set), MakeDefaultRanking());
+        auto service = MakeStaticNameService(LoadDefaultNameSet(), LoadFrequencyData());
         auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
         {
             TVector<TCandidate> expected = {
@@ -711,10 +710,8 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
     }
 
     Y_UNIT_TEST(NameNormalization) {
-        auto set = MakeDefaultNameSet();
-        auto service = MakeStaticNameService(std::move(set), MakeDefaultRanking());
+        auto service = MakeStaticNameService(LoadDefaultNameSet(), LoadFrequencyData());
         auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
-
         TVector<TCandidate> expected = {
             {HintName, "IGNORE_TYPE_V3"},
         };
@@ -749,7 +746,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
                 {"unordered", 2},
             },
         };
-        auto service = MakeStaticNameService(MakeDefaultNameSet(), MakeDefaultRanking(frequency));
+        auto service = MakeStaticNameService(
+            Pruned(LoadDefaultNameSet(), LoadFrequencyData()),
+            std::move(frequency));
         auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
         {
             TVector<TCandidate> expected = {