YQL-19747 Rank keywords just by plain usages

- [x] Rank keywords just by plain usages
- [x] `LocalSyntaxAnalysis` now returns a mapping `:: Keyword -> [Following Keywords]`.
- [x] Extracted keyword sequence formatting from `syntax/local` to `syntax/format`.
- [x] Extracted token display logic from `syntax/local` to `antlr4/vocabulary` as it is ANTLR dependent.

---

Example

```python
$ ./yql_complete <<< "select "
[Keyword] CAST(
[Keyword] NULL
[Keyword] NOT
[FunctionName] If(
[FunctionName] Yson::ConvertToString(
[FunctionName] Count(
[FunctionName] Sum(
[FunctionName] Unwrap(
[FunctionName] Coalesce(
[Keyword] DISTINCT
[Keyword] ALL
[Keyword] CASE
[FunctionName] Max(
[Keyword] FALSE
[FunctionName] Some(
```

---

- Related to https://github.com/ydb-platform/ydb/issues/9056
- Related to https://github.com/vityaman/ydb/issues/17

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1197
commit_hash:f42cb4aaffe6de7c9137069c4d9c635ee110a805

1 files changed, 20 insertions, 9 deletions

diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
index dd7145169c1..a0681b1888f 100644
--- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
@@ -324,6 +324,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "CURRENT_TIMESTAMP"},
             {Keyword, "DICT<"},
             {Keyword, "DISTINCT"},
+            {FunctionName, "DateTime::Split("},
             {Keyword, "EMPTY_ACTION"},
             {Keyword, "ENUM"},
             {Keyword, "EXISTS("},
@@ -340,12 +341,11 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "SET<"},
             {Keyword, "STREAM"},
             {Keyword, "STRUCT"},
+            {FunctionName, "StartsWith("},
             {Keyword, "TAGGED<"},
             {Keyword, "TRUE"},
             {Keyword, "TUPLE"},
             {Keyword, "VARIANT"},
-            {FunctionName, "DateTime::Split("},
-            {FunctionName, "StartsWith("},
         };
 
         auto engine = MakeSqlCompletionEngineUT();
@@ -362,6 +362,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "CURRENT_TIME"},
             {Keyword, "CURRENT_TIMESTAMP"},
             {Keyword, "DICT<"},
+            {FunctionName, "DateTime::Split("},
             {Keyword, "EMPTY_ACTION"},
             {Keyword, "ENUM"},
             {Keyword, "EXISTS("},
@@ -378,12 +379,11 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "SET<"},
             {Keyword, "STREAM<"},
             {Keyword, "STRUCT"},
+            {FunctionName, "StartsWith("},
             {Keyword, "TAGGED<"},
             {Keyword, "TRUE"},
             {Keyword, "TUPLE"},
             {Keyword, "VARIANT"},
-            {FunctionName, "DateTime::Split("},
-            {FunctionName, "StartsWith("},
         };
 
         auto engine = MakeSqlCompletionEngineUT();
@@ -415,8 +415,8 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "STRUCT"},
             {Keyword, "TAGGED<"},
             {Keyword, "TUPLE"},
-            {Keyword, "VARIANT<"},
             {TypeName, "Uint64"},
+            {Keyword, "VARIANT<"},
         };
 
         auto engine = MakeSqlCompletionEngineUT();
@@ -505,8 +505,8 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
     Y_UNIT_TEST(InsertTableHintName) {
         TVector<TCandidate> expected = {
             {Keyword, "COLUMNS"},
-            {Keyword, "SCHEMA"},
             {HintName, "EXPIRATION"},
+            {Keyword, "SCHEMA"},
         };
 
         auto engine = MakeSqlCompletionEngineUT();
@@ -614,7 +614,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
     Y_UNIT_TEST(OnFailingNameService) {
         auto service = MakeHolder<TFailingNameService>();
         auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
-        UNIT_ASSERT_NO_EXCEPTION(Complete(engine, {""}));
+        UNIT_ASSERT_EXCEPTION(Complete(engine, {""}), TDummyException);
         UNIT_ASSERT_EXCEPTION(Complete(engine, {"SELECT OPTIONAL<U"}), TDummyException);
         UNIT_ASSERT_EXCEPTION(Complete(engine, {"SELECT CAST (1 AS "}).size(), TDummyException);
     }
@@ -644,6 +644,10 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
 
     Y_UNIT_TEST(Ranking) {
         TFrequencyData frequency = {
+            .Keywords = {
+                {"select", 2},
+                {"insert", 4},
+            },
             .Pragmas = {
                 {"yt.defaultmemorylimit", 16},
                 {"yt.annotations", 8},
@@ -670,6 +674,13 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
         auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
         {
             TVector<TCandidate> expected = {
+                {Keyword, "INSERT"},
+                {Keyword, "SELECT"},
+            };
+            UNIT_ASSERT_VALUES_EQUAL(CompleteTop(expected.size(), engine, {""}), expected);
+        }
+        {
+            TVector<TCandidate> expected = {
                 {PragmaName, "DefaultMemoryLimit"},
                 {PragmaName, "Annotations"},
             };
@@ -701,10 +712,10 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
         }
         {
             TVector<TCandidate> expected = {
-                {Keyword, "COLUMNS"},
-                {Keyword, "SCHEMA"},
                 {HintName, "XLOCK"},
                 {HintName, "UNORDERED"},
+                {Keyword, "COLUMNS"},
+                {HintName, "FORCEINFERSCHEMA"},
             };
             UNIT_ASSERT_VALUES_EQUAL(CompleteTop(expected.size(), engine, {"SELECT * FROM a WITH "}), expected);
         }