YQL-19747 Rank keywords just by plain usages

- [x] Rank keywords just by plain usages
- [x] `LocalSyntaxAnalysis` now returns a mapping `:: Keyword -> [Following Keywords]`.
- [x] Extracted keyword sequence formatting from `syntax/local` to `syntax/format`.
- [x] Extracted token display logic from `syntax/local` to `antlr4/vocabulary` as it is ANTLR dependent.

---

Example

```python
$ ./yql_complete <<< "select "
[Keyword] CAST(
[Keyword] NULL
[Keyword] NOT
[FunctionName] If(
[FunctionName] Yson::ConvertToString(
[FunctionName] Count(
[FunctionName] Sum(
[FunctionName] Unwrap(
[FunctionName] Coalesce(
[Keyword] DISTINCT
[Keyword] ALL
[Keyword] CASE
[FunctionName] Max(
[Keyword] FALSE
[FunctionName] Some(
```

---

- Related to https://github.com/ydb-platform/ydb/issues/9056
- Related to https://github.com/vityaman/ydb/issues/17

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1197
commit_hash:f42cb4aaffe6de7c9137069c4d9c635ee110a805

1 files changed, 6 insertions, 17 deletions

diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp
index e6a7430ca27..4b6fac094d5 100644
--- a/yql/essentials/sql/v1/complete/syntax/local.cpp
+++ b/yql/essentials/sql/v1/complete/syntax/local.cpp
@@ -6,6 +6,7 @@
 
 #include <yql/essentials/sql/v1/complete/antlr4/c3i.h>
 #include <yql/essentials/sql/v1/complete/antlr4/c3t.h>
+#include <yql/essentials/sql/v1/complete/antlr4/vocabulary.h>
 
 #include <yql/essentials/core/issue/yql_issue.h>
 
@@ -115,34 +116,22 @@ namespace NSQLComplete {
             return true;
         }
 
-        TVector<TString> SiftedKeywords(const TC3Candidates& candidates) {
+        TLocalSyntaxContext::TKeywords SiftedKeywords(const TC3Candidates& candidates) {
             const auto& vocabulary = Grammar->GetVocabulary();
             const auto& keywordTokens = Grammar->GetKeywordTokens();
 
-            TVector<TString> keywords;
+            TLocalSyntaxContext::TKeywords keywords;
             for (const auto& token : candidates.Tokens) {
                 if (keywordTokens.contains(token.Number)) {
-                    keywords.emplace_back(Display(vocabulary, token.Number));
-                    for (auto following : token.Following) {
-                        if (keywordTokens.contains(following)) {
-                            keywords.back() += " ";
-                        }
-                        keywords.back() += Display(vocabulary, following);
+                    auto& following = keywords[Display(vocabulary, token.Number)];
+                    for (auto next : token.Following) {
+                        following.emplace_back(Display(vocabulary, next));
                     }
                 }
             }
             return keywords;
         }
 
-        std::string Display(const antlr4::dfa::Vocabulary& vocabulary, TTokenId tokenType) {
-            auto name = vocabulary.getDisplayName(tokenType);
-            if (2 <= name.length() && name.starts_with('\'') && name.ends_with('\'')) {
-                name.erase(static_cast<std::string::size_type>(0), 1);
-                name.pop_back();
-            }
-            return name;
-        }
-
         std::optional<TLocalSyntaxContext::TPragma> PragmaMatch(
             const NSQLTranslation::TParsedTokenList& tokens, const TC3Candidates& candidates) {
             if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyPragmaStack))) {