YQL-19747 Split statements

When we run completion engine on multi-statement query, where preceding statements are syntactically incorrect, `antlr4-c3` does not return candidates. Running engine only on a current statement provides a best-effort attempt to provide candidates.

- Related to https://github.com/ydb-platform/ydb/issues/9056
- Depends on https://github.com/ytsaurus/ytsaurus/pull/1127 (`ELexerFlavor`)

---

Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1144
commit_hash:0ced9443a9712191f5420246531f781ca4bc5f42

1 files changed, 35 insertions, 15 deletions

diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
index e0a012f9f6e..c65eba0e2d4 100644
--- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
@@ -1,5 +1,8 @@
 #include "sql_complete.h"
 
+#include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h>
+#include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h>
+
 #include <library/cpp/testing/unittest/registar.h>
 
 using namespace NSQLComplete;
@@ -7,6 +10,15 @@ using namespace NSQLComplete;
 Y_UNIT_TEST_SUITE(SqlCompleteTests) {
     using ECandidateKind::Keyword;
 
+    ISqlCompletionEngine::TPtr MakeSqlCompletionEngineUT() {
+        NSQLTranslationV1::TLexers lexers;
+        lexers.Antlr4Pure = NSQLTranslationV1::MakeAntlr4PureLexerFactory();
+        lexers.Antlr4PureAnsi = NSQLTranslationV1::MakeAntlr4PureAnsiLexerFactory();
+        return MakeSqlCompletionEngine([lexers = std::move(lexers)](bool ansi) {
+            return NSQLTranslationV1::MakeLexer(lexers, ansi, /* antlr4 = */ true, /* pure = */ true);
+        });
+    }
+
     TVector<TCandidate> Complete(ISqlCompletionEngine::TPtr& engine, TStringBuf prefix) {
         return engine->Complete({prefix}).Candidates;
     }
@@ -50,7 +62,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "VALUES"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {""}), expected);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {" "}), expected);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"  "}), expected);
@@ -76,7 +88,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "USER"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"ALTER "}), expected);
     }
 
@@ -99,7 +111,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "VIEW"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"CREATE "}), expected);
     }
 
@@ -108,7 +120,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "FROM"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"DELETE "}), expected);
     }
 
@@ -128,7 +140,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "VIEW"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"DROP "}), expected);
     }
 
@@ -171,7 +183,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "VALUES"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"EXPLAIN "}), expected);
     }
 
@@ -196,7 +208,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "USE"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"GRANT "}), expected);
     }
 
@@ -206,7 +218,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "OR"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"INSERT "}), expected);
     }
 
@@ -227,7 +239,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "VARIANT"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"PRAGMA "}), expected);
     }
 
@@ -265,7 +277,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "VARIANT"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT "}), expected);
     }
 
@@ -275,18 +287,18 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "OBJECT"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"UPSERT "}), expected);
     }
 
     Y_UNIT_TEST(UTF8Wide) {
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"\xF0\x9F\x98\x8A"}).size(), 0);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"编码"}).size(), 0);
     }
 
     Y_UNIT_TEST(WordBreak) {
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT ("}).size(), 28);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT (1)"}).size(), 30);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT 1;"}).size(), 35);
@@ -300,7 +312,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             "  Bool(field), Math::Sin(var) \n"
             "FROM `local/test/space/table` JOIN test;");
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
 
         for (std::size_t size = 0; size <= queryUtf16.size(); ++size) {
             const TWtringBuf prefixUtf16(queryUtf16, 0, size);
@@ -314,10 +326,18 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
             {Keyword, "SELECT"},
         };
 
-        auto engine = MakeSqlCompletionEngine();
+        auto engine = MakeSqlCompletionEngineUT();
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "se"), expected);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "sE"), expected);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "Se"), expected);
         UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SE"), expected);
     }
+
+    Y_UNIT_TEST(InvalidStatementsRecovery) {
+        auto engine = MakeSqlCompletionEngineUT();
+        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "select select; ").size(), 35);
+        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "select select;").size(), 35);
+        UNIT_ASSERT_VALUES_EQUAL_C(Complete(engine, "!;").size(), 0, "Lexer failing");
+    }
+
 } // Y_UNIT_TEST_SUITE(SqlCompleteTests)