diff options
author | vityaman <vityaman.dev@yandex.ru> | 2025-04-08 16:11:56 +0300 |
---|---|---|
committer | robot-piglet <robot-piglet@yandex-team.com> | 2025-04-08 16:26:47 +0300 |
commit | 2f90258cf6f1625ba0c99f6f37b6c9f336590534 (patch) | |
tree | 87891b9f23d6c25e134671a920553864a5331e7f | |
parent | 7077d90968fe79dfe126e92543d669259f02ef3a (diff) | |
download | ydb-2f90258cf6f1625ba0c99f6f37b6c9f336590534.tar.gz |
YQL-19747 Complete after PRAGMA and multi-token names
- [x] Complete after PRAGMA
- [x] Complete multi-token names correctly, for example, `yt.` returns only `DisableStrict`, not `yt.DisableStrict` and `DateTime::` returns `Split`, not `DateTime::Split`.
I tried to implement it using `CompletedToken` edition, but not all completion environments support candidates with various `contextLen` (`Replxx` does not). So I decided that completions should rewrite only the current token, not sequences. For example, on `DateTime::Spl` rewrite only `Spl`. It makes sense as multi-token names have some namespace separated by a punctuation, so used types only namespace and gets names inside of it.
---
Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1181
commit_hash:9d8967ac43b9348f6dbb53837d92a9dcc9b51f48
15 files changed, 286 insertions, 30 deletions
diff --git a/yql/essentials/sql/v1/complete/name/name_service.h b/yql/essentials/sql/v1/complete/name/name_service.h index d3cf153341e..6a7e38dfc20 100644 --- a/yql/essentials/sql/v1/complete/name/name_service.h +++ b/yql/essentials/sql/v1/complete/name/name_service.h @@ -13,28 +13,40 @@ namespace NSQLComplete { TString Indentifier; }; + struct TNamespaced { + TString Namespace; + }; + + struct TPragmaName: TIndentifier { + struct TConstraints: TNamespaced {}; + }; + struct TTypeName: TIndentifier { using TConstraints = std::monostate; }; struct TFunctionName: TIndentifier { - using TConstraints = std::monostate; + struct TConstraints: TNamespaced {}; }; using TGenericName = std::variant< + TPragmaName, TTypeName, TFunctionName>; struct TNameRequest { struct { - std::optional<TTypeName::TConstraints> TypeName; - std::optional<TTypeName::TConstraints> Function; + std::optional<TPragmaName::TConstraints> Pragma; + std::optional<TTypeName::TConstraints> Type; + std::optional<TFunctionName::TConstraints> Function; } Constraints; TString Prefix = ""; size_t Limit = 128; bool IsEmpty() const { - return !Constraints.TypeName && !Constraints.Function; + return !Constraints.Pragma && + !Constraints.Type && + !Constraints.Function; } }; diff --git a/yql/essentials/sql/v1/complete/name/static/frequency.cpp b/yql/essentials/sql/v1/complete/name/static/frequency.cpp index d9c8ba9652c..2de082e287e 100644 --- a/yql/essentials/sql/v1/complete/name/static/frequency.cpp +++ b/yql/essentials/sql/v1/complete/name/static/frequency.cpp @@ -14,6 +14,7 @@ namespace NSQLComplete { const char* Sum = "sum"; } Key; struct { + const char* Pragma = "PRAGMA"; const char* Type = "TYPE"; const char* Func = "FUNC"; const char* Module = "MODULE"; @@ -53,14 +54,17 @@ namespace NSQLComplete { TFrequencyData Convert(TVector<TFrequencyItem> items) { TFrequencyData data; for (auto& item : items) { - if (item.Parent == Json.Parent.Type || + if (item.Parent == Json.Parent.Pragma || + item.Parent == Json.Parent.Type || item.Parent == Json.Parent.Func || item.Parent == Json.Parent.ModuleFunc || item.Parent == Json.Parent.Module) { item.Rule = ToLowerUTF8(item.Rule); } - if (item.Parent == Json.Parent.Type) { + if (item.Parent == Json.Parent.Pragma) { + data.Pragmas[item.Rule] += item.Sum; + } else if (item.Parent == Json.Parent.Type) { data.Types[item.Rule] += item.Sum; } else if (item.Parent == Json.Parent.Func || item.Parent == Json.Parent.ModuleFunc) { diff --git a/yql/essentials/sql/v1/complete/name/static/frequency.h b/yql/essentials/sql/v1/complete/name/static/frequency.h index 3d128f824b4..067453bc404 100644 --- a/yql/essentials/sql/v1/complete/name/static/frequency.h +++ b/yql/essentials/sql/v1/complete/name/static/frequency.h @@ -6,6 +6,7 @@ namespace NSQLComplete { struct TFrequencyData { + THashMap<TString, size_t> Pragmas; THashMap<TString, size_t> Types; THashMap<TString, size_t> Functions; }; diff --git a/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp b/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp index 29c303b3102..9fdf314fee6 100644 --- a/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp +++ b/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp @@ -26,6 +26,10 @@ namespace NSQLComplete { return keys; } + TVector<TString> ParsePragmas(NJson::TJsonValue json) { + return ParseNames(json.GetArraySafe()); + } + TVector<TString> ParseTypes(NJson::TJsonValue json) { return ParseNames(json.GetArraySafe()); } @@ -48,6 +52,7 @@ namespace NSQLComplete { NameSet MakeDefaultNameSet() { return { + .Pragmas = ParsePragmas(LoadJsonResource("pragmas_opensource.json")), .Types = ParseTypes(LoadJsonResource("types.json")), .Functions = Merge( ParseFunctions(LoadJsonResource("sql_functions.json")), diff --git a/yql/essentials/sql/v1/complete/name/static/name_service.cpp b/yql/essentials/sql/v1/complete/name/static/name_service.cpp index fdb1dd4eae1..42769198579 100644 --- a/yql/essentials/sql/v1/complete/name/static/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/static/name_service.cpp @@ -35,12 +35,43 @@ namespace NSQLComplete { } } + TString Prefixed(const TStringBuf requestPrefix, const TStringBuf delimeter, const TNamespaced& namespaced) { + TString prefix; + if (!namespaced.Namespace.empty()) { + prefix += namespaced.Namespace; + prefix += delimeter; + } + prefix += requestPrefix; + return prefix; + } + + void FixPrefix(TString& name, const TStringBuf delimeter, const TNamespaced& namespaced) { + if (namespaced.Namespace.empty()) { + return; + } + name.remove(0, namespaced.Namespace.size() + delimeter.size()); + } + + void FixPrefix(TGenericName& name, const TNameRequest& request) { + std::visit([&](auto& name) -> size_t { + using T = std::decay_t<decltype(name)>; + if constexpr (std::is_same_v<T, TPragmaName>) { + FixPrefix(name.Indentifier, ".", *request.Constraints.Pragma); + } + if constexpr (std::is_same_v<T, TFunctionName>) { + FixPrefix(name.Indentifier, "::", *request.Constraints.Function); + } + return 0; + }, name); + } + class TStaticNameService: public INameService { public: explicit TStaticNameService(NameSet names, IRanking::TPtr ranking) : NameSet_(std::move(names)) , Ranking_(std::move(ranking)) { + Sort(NameSet_.Pragmas, NoCaseCompare); Sort(NameSet_.Types, NoCaseCompare); Sort(NameSet_.Functions, NoCaseCompare); } @@ -48,20 +79,30 @@ namespace NSQLComplete { TFuture<TNameResponse> Lookup(TNameRequest request) override { TNameResponse response; - if (request.Constraints.TypeName) { + if (request.Constraints.Pragma) { + auto prefix = Prefixed(request.Prefix, ".", *request.Constraints.Pragma); + auto names = FilteredByPrefix(prefix, NameSet_.Pragmas); + AppendAs<TPragmaName>(response.RankedNames, names); + } + + if (request.Constraints.Type) { AppendAs<TTypeName>( response.RankedNames, FilteredByPrefix(request.Prefix, NameSet_.Types)); } if (request.Constraints.Function) { - AppendAs<TFunctionName>( - response.RankedNames, - FilteredByPrefix(request.Prefix, NameSet_.Functions)); + auto prefix = Prefixed(request.Prefix, "::", *request.Constraints.Function); + auto names = FilteredByPrefix(prefix, NameSet_.Functions); + AppendAs<TFunctionName>(response.RankedNames, names); } Ranking_->CropToSortedPrefix(response.RankedNames, request.Limit); + for (auto& name : response.RankedNames) { + FixPrefix(name, request); + } + return NThreading::MakeFuture(std::move(response)); } diff --git a/yql/essentials/sql/v1/complete/name/static/name_service.h b/yql/essentials/sql/v1/complete/name/static/name_service.h index a5c90465c83..348d6ece454 100644 --- a/yql/essentials/sql/v1/complete/name/static/name_service.h +++ b/yql/essentials/sql/v1/complete/name/static/name_service.h @@ -7,6 +7,7 @@ namespace NSQLComplete { struct NameSet { + TVector<TString> Pragmas; TVector<TString> Types; TVector<TString> Functions; }; diff --git a/yql/essentials/sql/v1/complete/name/static/ranking.cpp b/yql/essentials/sql/v1/complete/name/static/ranking.cpp index 45e6e2b2fa2..b3d5c3c8c31 100644 --- a/yql/essentials/sql/v1/complete/name/static/ranking.cpp +++ b/yql/essentials/sql/v1/complete/name/static/ranking.cpp @@ -59,6 +59,12 @@ namespace NSQLComplete { auto identifier = ToLowerUTF8(ContentView(name)); + if constexpr (std::is_same_v<T, TPragmaName>) { + if (auto weight = Frequency_.Pragmas.FindPtr(identifier)) { + return *weight; + } + } + if constexpr (std::is_same_v<T, TFunctionName>) { if (auto weight = Frequency_.Functions.FindPtr(identifier)) { return *weight; diff --git a/yql/essentials/sql/v1/complete/name/static/ya.make b/yql/essentials/sql/v1/complete/name/static/ya.make index 639371447af..3c6547f3b52 100644 --- a/yql/essentials/sql/v1/complete/name/static/ya.make +++ b/yql/essentials/sql/v1/complete/name/static/ya.make @@ -13,6 +13,7 @@ PEERDIR( ) RESOURCE( + yql/essentials/data/language/pragmas_opensource.json pragmas_opensource.json yql/essentials/data/language/types.json types.json yql/essentials/data/language/sql_functions.json sql_functions.json yql/essentials/data/language/udfs_basic.json udfs_basic.json diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp index ed3afa29df4..85fcf87afd1 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete.cpp @@ -35,10 +35,10 @@ namespace NSQLComplete { << " for input size " << input.Text.size(); } - auto prefix = input.Text.Head(input.CursorPosition); - auto completedToken = GetCompletedToken(prefix); + TLocalSyntaxContext context = SyntaxAnalysis->Analyze(input); - auto context = SyntaxAnalysis->Analyze(input); + TStringBuf prefix = input.Text.Head(input.CursorPosition); + TCompletedToken completedToken = GetCompletedToken(prefix); TVector<TCandidate> candidates; EnrichWithKeywords(candidates, std::move(context.Keywords), completedToken); @@ -85,12 +85,20 @@ namespace NSQLComplete { .Limit = Configuration.Limit - candidates.size(), }; + if (context.Pragma) { + TPragmaName::TConstraints constraints; + constraints.Namespace = context.Pragma->Namespace; + request.Constraints.Pragma = std::move(constraints); + } + if (context.IsTypeName) { - request.Constraints.TypeName = TTypeName::TConstraints(); + request.Constraints.Type = TTypeName::TConstraints(); } - if (context.IsFunctionName) { - request.Constraints.Function = TFunctionName::TConstraints(); + if (context.Function) { + TFunctionName::TConstraints constraints; + constraints.Namespace = context.Function->Namespace; + request.Constraints.Function = std::move(constraints); } if (request.IsEmpty()) { @@ -107,6 +115,9 @@ namespace NSQLComplete { for (auto& name : names) { candidates.emplace_back(std::visit([](auto&& name) -> TCandidate { using T = std::decay_t<decltype(name)>; + if constexpr (std::is_base_of_v<TPragmaName, T>) { + return {ECandidateKind::PragmaName, std::move(name.Indentifier)}; + } if constexpr (std::is_base_of_v<TTypeName, T>) { return {ECandidateKind::TypeName, std::move(name.Indentifier)}; } @@ -162,6 +173,9 @@ void Out<NSQLComplete::ECandidateKind>(IOutputStream& out, NSQLComplete::ECandid case NSQLComplete::ECandidateKind::Keyword: out << "Keyword"; break; + case NSQLComplete::ECandidateKind::PragmaName: + out << "PragmaName"; + break; case NSQLComplete::ECandidateKind::TypeName: out << "TypeName"; break; diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h index b8a970efd8f..5d0271b4dca 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.h +++ b/yql/essentials/sql/v1/complete/sql_complete.h @@ -20,6 +20,7 @@ namespace NSQLComplete { enum class ECandidateKind { Keyword, + PragmaName, TypeName, FunctionName, }; diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp index e9f5dbdfb73..7d595842afb 100644 --- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -39,6 +39,7 @@ public: Y_UNIT_TEST_SUITE(SqlCompleteTests) { using ECandidateKind::FunctionName; using ECandidateKind::Keyword; + using ECandidateKind::PragmaName; using ECandidateKind::TypeName; TLexerSupplier MakePureLexerSupplier() { @@ -55,8 +56,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { ISqlCompletionEngine::TPtr MakeSqlCompletionEngineUT() { TLexerSupplier lexer = MakePureLexerSupplier(); NameSet names = { + .Pragmas = {"yson.CastToString"}, .Types = {"Uint64"}, - .Functions = {"StartsWith"}, + .Functions = {"StartsWith", "DateTime::Split"}, }; auto ranking = MakeDefaultRanking({}); INameService::TPtr service = MakeStaticNameService(std::move(names), std::move(ranking)); @@ -267,12 +269,36 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } Y_UNIT_TEST(Pragma) { - TVector<TCandidate> expected = { - {Keyword, "ANSI"}, - }; - auto engine = MakeSqlCompletionEngineUT(); - UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"PRAGMA "}), expected); + { + TVector<TCandidate> expected = { + {Keyword, "ANSI"}, + {PragmaName, "yson.CastToString"}}; + auto completion = engine->Complete({"PRAGMA "}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, ""); + } + { + TVector<TCandidate> expected = { + {PragmaName, "yson.CastToString"}}; + auto completion = engine->Complete({"PRAGMA yson"}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "yson"); + } + { + TVector<TCandidate> expected = { + {PragmaName, "CastToString"}}; + auto completion = engine->Complete({"PRAGMA yson."}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, ""); + } + { + TVector<TCandidate> expected = { + {PragmaName, "CastToString"}}; + auto completion = engine->Complete({"PRAGMA yson.cast"}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "cast"); + } } Y_UNIT_TEST(Select) { @@ -307,6 +333,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { {Keyword, "TRUE"}, {Keyword, "TUPLE"}, {Keyword, "VARIANT"}, + {FunctionName, "DateTime::Split("}, {FunctionName, "StartsWith("}, }; @@ -344,6 +371,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { {Keyword, "TRUE"}, {Keyword, "TUPLE"}, {Keyword, "VARIANT"}, + {FunctionName, "DateTime::Split("}, {FunctionName, "StartsWith("}, }; @@ -402,6 +430,41 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } } + Y_UNIT_TEST(FunctionName) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {FunctionName, "DateTime::Split("}, + }; + auto completion = engine->Complete({"SELECT Date"}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "Date"); + } + { + TVector<TCandidate> expected = { + {FunctionName, "Split("}, + }; + auto completion = engine->Complete({"SELECT DateTime:"}); + UNIT_ASSERT(completion.Candidates.empty()); + } + { + TVector<TCandidate> expected = { + {FunctionName, "Split("}, + }; + auto completion = engine->Complete({"SELECT DateTime::"}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, ""); + } + { + TVector<TCandidate> expected = { + {FunctionName, "Split("}, + }; + auto completion = engine->Complete({"SELECT DateTime::s"}); + UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "s"); + } + } + Y_UNIT_TEST(UTF8Wide) { auto engine = MakeSqlCompletionEngineUT(); UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"\xF0\x9F\x98\x8A"}).size(), 0); @@ -410,9 +473,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { Y_UNIT_TEST(WordBreak) { auto engine = MakeSqlCompletionEngineUT(); - UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT ("}).size(), 29); - UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT (1)"}).size(), 30); - UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT 1;"}).size(), 35); + UNIT_ASSERT_GE(Complete(engine, {"SELECT ("}).size(), 29); + UNIT_ASSERT_GE(Complete(engine, {"SELECT (1)"}).size(), 30); + UNIT_ASSERT_GE(Complete(engine, {"SELECT 1;"}).size(), 35); } Y_UNIT_TEST(Typing) { @@ -481,6 +544,16 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT OPTIONAL<U"}), expected); } + { + TVector<TCandidate> expected = { + {PragmaName, "yson.DisableStrict"}, + {PragmaName, "yson.AutoConvert"}, + {PragmaName, "yson.Strict"}, + {PragmaName, "yson.CastToString"}, + {PragmaName, "yson.DisableCastToString"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"PRAGMA yson"}), expected); + } } Y_UNIT_TEST(OnFailingNameService) { @@ -516,6 +589,10 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { Y_UNIT_TEST(Ranking) { TFrequencyData frequency = { + .Pragmas = { + {"yt.defaultmemorylimit", 16}, + {"yt.annotations", 8}, + }, .Types = { {"int32", 128}, {"int64", 64}, @@ -533,6 +610,15 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { auto service = MakeStaticNameService(MakeDefaultNameSet(), MakeDefaultRanking(frequency)); auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service)); { + TVector<TCandidate> expectedPrefix = { + {PragmaName, "DefaultMemoryLimit"}, + {PragmaName, "Annotations"}, + }; + auto actualPrefix = Complete(engine, {"PRAGMA yt."}); + actualPrefix.crop(expectedPrefix.size()); + UNIT_ASSERT_VALUES_EQUAL(actualPrefix, expectedPrefix); + } + { TVector<TCandidate> expected = { {TypeName, "Int32"}, {TypeName, "Int64"}, diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp index cac43e5a320..28ace474f8e 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.cpp +++ b/yql/essentials/sql/v1/complete/syntax/local.cpp @@ -53,10 +53,14 @@ namespace NSQLComplete { } auto candidates = C3.Complete(prefix); + + NSQLTranslation::TParsedTokenList tokens = Tokenized(prefix); + return { .Keywords = SiftedKeywords(candidates), + .Pragma = PragmaMatch(tokens, candidates), .IsTypeName = IsTypeNameMatched(candidates), - .IsFunctionName = IsFunctionNameMatched(candidates), + .Function = FunctionMatch(tokens, candidates), }; } @@ -131,16 +135,73 @@ namespace NSQLComplete { return name; } + std::optional<TLocalSyntaxContext::TPragma> PragmaMatch( + const NSQLTranslation::TParsedTokenList& tokens, const TC3Candidates& candidates) { + bool isMatched = AnyOf(candidates.Rules, [&](const TMatchedRule& rule) { + return IsLikelyPragmaStack(rule.ParserCallStack); + }); + if (!isMatched) { + return std::nullopt; + } + + TLocalSyntaxContext::TPragma pragma; + if (EndsWith(tokens, {"ID_PLAIN", "DOT"})) { + pragma.Namespace = tokens[tokens.size() - 2].Content; + } else if (EndsWith(tokens, {"ID_PLAIN", "DOT", ""})) { + pragma.Namespace = tokens[tokens.size() - 3].Content; + } + return pragma; + } + bool IsTypeNameMatched(const TC3Candidates& candidates) { return AnyOf(candidates.Rules, [&](const TMatchedRule& rule) { return IsLikelyTypeStack(rule.ParserCallStack); }); } - bool IsFunctionNameMatched(const TC3Candidates& candidates) { - return AnyOf(candidates.Rules, [&](const TMatchedRule& rule) { + std::optional<TLocalSyntaxContext::TFunction> FunctionMatch( + const NSQLTranslation::TParsedTokenList& tokens, const TC3Candidates& candidates) { + bool isMatched = AnyOf(candidates.Rules, [&](const TMatchedRule& rule) { return IsLikelyFunctionStack(rule.ParserCallStack); }); + if (!isMatched) { + return std::nullopt; + } + + TLocalSyntaxContext::TFunction function; + if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE"})) { + function.Namespace = tokens[tokens.size() - 2].Content; + } else if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE", ""})) { + function.Namespace = tokens[tokens.size() - 3].Content; + } + return function; + } + + NSQLTranslation::TParsedTokenList Tokenized(const TStringBuf text) { + NSQLTranslation::TParsedTokenList tokens; + NYql::TIssues issues; + if (!NSQLTranslation::Tokenize( + *Lexer_, TString(text), /* queryName = */ "", + tokens, issues, /* maxErrors = */ 0)) { + return {}; + } + Y_ENSURE(!tokens.empty() && tokens.back().Name == "EOF"); + tokens.pop_back(); + return tokens; + } + + bool EndsWith( + const NSQLTranslation::TParsedTokenList& tokens, + const TVector<TStringBuf>& pattern) { + if (tokens.size() < pattern.size()) { + return false; + } + for (yssize_t i = tokens.ysize() - 1, j = pattern.ysize() - 1; 0 <= j; --i, --j) { + if (!pattern[j].empty() && tokens[i].Name != pattern[j]) { + return false; + } + } + return true; } const ISqlGrammar* Grammar; diff --git a/yql/essentials/sql/v1/complete/syntax/local.h b/yql/essentials/sql/v1/complete/syntax/local.h index 79984d00e2b..24e78108e10 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.h +++ b/yql/essentials/sql/v1/complete/syntax/local.h @@ -10,9 +10,18 @@ namespace NSQLComplete { struct TLocalSyntaxContext { + struct TPragma { + TString Namespace; + }; + + struct TFunction { + TString Namespace; + }; + TVector<TString> Keywords; + std::optional<TPragma> Pragma; bool IsTypeName; - bool IsFunctionName; + std::optional<TFunction> Function; }; class ILocalSyntaxAnalysis { diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp index 57e058fa900..1bfcac47266 100644 --- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp +++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp @@ -24,6 +24,11 @@ namespace NSQLComplete { RULE(Keyword_compat), }; + const TVector<TRuleId> PragmaNameRules = { + RULE(Opt_id_prefix_or_type), + RULE(An_id), + }; + const TVector<TRuleId> TypeNameRules = { RULE(Type_name_simple), RULE(An_id_or_type), @@ -62,6 +67,11 @@ namespace NSQLComplete { return Find(stack, rule) != std::end(stack); } + bool IsLikelyPragmaStack(const TParserCallStack& stack) { + return EndsWith({RULE(Pragma_stmt), RULE(Opt_id_prefix_or_type)}, stack) || + EndsWith({RULE(Pragma_stmt), RULE(An_id)}, stack); + } + bool IsLikelyTypeStack(const TParserCallStack& stack) { return EndsWith({RULE(Type_name_simple)}, stack) || (Contains({RULE(Invoke_expr), @@ -75,12 +85,14 @@ namespace NSQLComplete { return EndsWith({RULE(Unary_casual_subexpr), RULE(Id_expr)}, stack) || EndsWith({RULE(Unary_casual_subexpr), RULE(Atom_expr), - RULE(An_id_or_type)}, stack); + RULE(An_id_or_type)}, stack) || + EndsWith({RULE(Atom_expr), RULE(Id_or_type)}, stack); } std::unordered_set<TRuleId> GetC3PreferredRules() { std::unordered_set<TRuleId> preferredRules; preferredRules.insert(std::begin(KeywordRules), std::end(KeywordRules)); + preferredRules.insert(std::begin(PragmaNameRules), std::end(PragmaNameRules)); preferredRules.insert(std::begin(TypeNameRules), std::end(TypeNameRules)); preferredRules.insert(std::begin(FunctionNameRules), std::end(FunctionNameRules)); return preferredRules; diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h index 756586988db..94533bddaae 100644 --- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h +++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h @@ -4,6 +4,8 @@ namespace NSQLComplete { + bool IsLikelyPragmaStack(const TParserCallStack& stack); + bool IsLikelyTypeStack(const TParserCallStack& stack); bool IsLikelyFunctionStack(const TParserCallStack& stack); |