YQL-19747: Fix caret token position

Outcomes:
1. Need to think more deeply about library interfaces. If you think that an interface is redundant, think again, because author knows a usecase that you do not imagine.
2. If you are lazy now to implement something robustly, think twice because the day will come and this code
will fire.

The problem was that on input ``` ... cluster.`/yql/t#` ``` C3 receives only a prefix ``` ... cluster.`/yql/t ``` and a token stream produced was incorrect and therefore completions too.

---

- Related to `YQL-19747`
- Related to https://github.com/ytsaurus/ytsaurus/pull/1257
- Related to https://github.com/ydb-platform/ydb/issues/9056

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1271
commit_hash:8bf5f2683c9e41963003da923f255f63b898aac5

5 files changed, 30 insertions, 16 deletions

diff --git a/yql/essentials/sql/v1/complete/antlr4/c3i.h b/yql/essentials/sql/v1/complete/antlr4/c3i.h
index 74c7805387f..ed69f3cf29b 100644
--- a/yql/essentials/sql/v1/complete/antlr4/c3i.h
+++ b/yql/essentials/sql/v1/complete/antlr4/c3i.h
@@ -38,7 +38,7 @@ namespace NSQLComplete {
             std::unordered_set<TRuleId> PreferredRules;
         };
 
-        virtual TC3Candidates Complete(TCompletionInput input) = 0;
+        virtual TC3Candidates Complete(TStringBuf text, size_t caretTokenIndex) = 0;
         virtual ~IC3Engine() = default;
     };
 
diff --git a/yql/essentials/sql/v1/complete/antlr4/c3t.h b/yql/essentials/sql/v1/complete/antlr4/c3t.h
index aca5ebf92e5..fb26c8c7f4a 100644
--- a/yql/essentials/sql/v1/complete/antlr4/c3t.h
+++ b/yql/essentials/sql/v1/complete/antlr4/c3t.h
@@ -40,10 +40,8 @@ namespace NSQLComplete {
             CompletionCore_.preferredRules = std::move(config.PreferredRules);
         }
 
-        TC3Candidates Complete(TCompletionInput input) override {
-            auto prefix = input.Text.Head(input.CursorPosition);
-            Assign(prefix);
-            const auto caretTokenIndex = CaretTokenIndex(prefix);
+        TC3Candidates Complete(TStringBuf text, size_t caretTokenIndex) override {
+            Assign(text);
             auto candidates = CompletionCore_.collectCandidates(caretTokenIndex);
             return Converted(std::move(candidates));
         }
@@ -56,14 +54,6 @@ namespace NSQLComplete {
             Tokens_.fill();
         }
 
-        size_t CaretTokenIndex(TStringBuf prefix) {
-            const auto tokensCount = Tokens_.size();
-            if (2 <= tokensCount && !LastWord(prefix).Empty()) {
-                return tokensCount - 2;
-            }
-            return tokensCount - 1;
-        }
-
         static TC3Candidates Converted(c3::CandidatesCollection candidates) {
             TC3Candidates converted;
             for (auto& [token, following] : candidates.tokens) {
diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
index a72446779cf..10afd08f827 100644
--- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
@@ -91,7 +91,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
                   {"/test/service/", {{"Table", "example"}}},
                   {"/.sys/", {{"Table", "status"}}}}},
             {"example",
-             {{"/", {{"Table", "people"}}}}},
+             {{"/", {{"Table", "people"},
+                     {"Folder", "yql"}}},
+              {"/yql/", {{"Table", "tutorial"}}}}},
             {"yt:saurus",
              {{"/", {{"Table", "maxim"}}}}},
         };
@@ -609,6 +611,12 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             };
             UNIT_ASSERT_VALUES_EQUAL(CompleteTop(1, engine, "SELECT * FROM example."), expected);
         }
+        {
+            TVector<TCandidate> expected = {
+                {TableName, "tutorial"},
+            };
+            UNIT_ASSERT_VALUES_EQUAL(CompleteTop(1, engine, "SELECT * FROM example.`/yql/t#`"), expected);
+        }
     }
 
     Y_UNIT_TEST(SelectWhere) {
diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp
index 549208d4cab..c20b67d2042 100644
--- a/yql/essentials/sql/v1/complete/syntax/local.cpp
+++ b/yql/essentials/sql/v1/complete/syntax/local.cpp
@@ -68,7 +68,7 @@ namespace NSQLComplete {
                 return {};
             }
 
-            TC3Candidates candidates = C3_.Complete(statement);
+            TC3Candidates candidates = C3Complete(statement, context);
 
             TLocalSyntaxContext result;
 
@@ -118,6 +118,23 @@ namespace NSQLComplete {
             return GetC3PreferredRules();
         }
 
+        TC3Candidates C3Complete(TCompletionInput statement, const TCursorTokenContext& context) {
+            auto enclosing = context.Enclosing();
+
+            size_t caretTokenIndex = context.Cursor.NextTokenIndex;
+            if (enclosing.Defined()) {
+                caretTokenIndex = enclosing->Index;
+            }
+
+            TStringBuf text = statement.Text;
+            if (enclosing.Defined() && enclosing->Base->Name == "NOT_EQUALS2") {
+                text = statement.Text.Head(statement.CursorPosition);
+                caretTokenIndex += 1;
+            }
+
+            return C3_.Complete(text, caretTokenIndex);
+        }
+
         TLocalSyntaxContext::TKeywords SiftedKeywords(const TC3Candidates& candidates) const {
             const auto& vocabulary = Grammar_->GetVocabulary();
             const auto& keywordTokens = Grammar_->GetKeywordTokens();
diff --git a/yql/essentials/sql/v1/complete/ya.make b/yql/essentials/sql/v1/complete/ya.make
index 010c8ac5243..f38e4bba319 100644
--- a/yql/essentials/sql/v1/complete/ya.make
+++ b/yql/essentials/sql/v1/complete/ya.make
@@ -10,7 +10,6 @@ PEERDIR(
     yql/essentials/sql/v1/complete/name/service
     yql/essentials/sql/v1/complete/syntax
     yql/essentials/sql/v1/complete/text
-
     # TODO(YQL-19747): add it to YDB CLI PEERDIR
     yql/essentials/sql/v1/complete/name/service/static
 )