aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvityaman <vityaman.dev@yandex.ru>2025-04-08 16:11:56 +0300
committerrobot-piglet <robot-piglet@yandex-team.com>2025-04-08 16:26:47 +0300
commit2f90258cf6f1625ba0c99f6f37b6c9f336590534 (patch)
tree87891b9f23d6c25e134671a920553864a5331e7f
parent7077d90968fe79dfe126e92543d669259f02ef3a (diff)
downloadydb-2f90258cf6f1625ba0c99f6f37b6c9f336590534.tar.gz
YQL-19747 Complete after PRAGMA and multi-token names
- [x] Complete after PRAGMA - [x] Complete multi-token names correctly, for example, `yt.` returns only `DisableStrict`, not `yt.DisableStrict` and `DateTime::` returns `Split`, not `DateTime::Split`. I tried to implement it using `CompletedToken` edition, but not all completion environments support candidates with various `contextLen` (`Replxx` does not). So I decided that completions should rewrite only the current token, not sequences. For example, on `DateTime::Spl` rewrite only `Spl`. It makes sense as multi-token names have some namespace separated by a punctuation, so used types only namespace and gets names inside of it. --- Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1181 commit_hash:9d8967ac43b9348f6dbb53837d92a9dcc9b51f48
-rw-r--r--yql/essentials/sql/v1/complete/name/name_service.h20
-rw-r--r--yql/essentials/sql/v1/complete/name/static/frequency.cpp8
-rw-r--r--yql/essentials/sql/v1/complete/name/static/frequency.h1
-rw-r--r--yql/essentials/sql/v1/complete/name/static/json_name_set.cpp5
-rw-r--r--yql/essentials/sql/v1/complete/name/static/name_service.cpp49
-rw-r--r--yql/essentials/sql/v1/complete/name/static/name_service.h1
-rw-r--r--yql/essentials/sql/v1/complete/name/static/ranking.cpp6
-rw-r--r--yql/essentials/sql/v1/complete/name/static/ya.make1
-rw-r--r--yql/essentials/sql/v1/complete/sql_complete.cpp26
-rw-r--r--yql/essentials/sql/v1/complete/sql_complete.h1
-rw-r--r--yql/essentials/sql/v1/complete/sql_complete_ut.cpp104
-rw-r--r--yql/essentials/sql/v1/complete/syntax/local.cpp67
-rw-r--r--yql/essentials/sql/v1/complete/syntax/local.h11
-rw-r--r--yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp14
-rw-r--r--yql/essentials/sql/v1/complete/syntax/parser_call_stack.h2
15 files changed, 286 insertions, 30 deletions
diff --git a/yql/essentials/sql/v1/complete/name/name_service.h b/yql/essentials/sql/v1/complete/name/name_service.h
index d3cf153341e..6a7e38dfc20 100644
--- a/yql/essentials/sql/v1/complete/name/name_service.h
+++ b/yql/essentials/sql/v1/complete/name/name_service.h
@@ -13,28 +13,40 @@ namespace NSQLComplete {
TString Indentifier;
};
+ struct TNamespaced {
+ TString Namespace;
+ };
+
+ struct TPragmaName: TIndentifier {
+ struct TConstraints: TNamespaced {};
+ };
+
struct TTypeName: TIndentifier {
using TConstraints = std::monostate;
};
struct TFunctionName: TIndentifier {
- using TConstraints = std::monostate;
+ struct TConstraints: TNamespaced {};
};
using TGenericName = std::variant<
+ TPragmaName,
TTypeName,
TFunctionName>;
struct TNameRequest {
struct {
- std::optional<TTypeName::TConstraints> TypeName;
- std::optional<TTypeName::TConstraints> Function;
+ std::optional<TPragmaName::TConstraints> Pragma;
+ std::optional<TTypeName::TConstraints> Type;
+ std::optional<TFunctionName::TConstraints> Function;
} Constraints;
TString Prefix = "";
size_t Limit = 128;
bool IsEmpty() const {
- return !Constraints.TypeName && !Constraints.Function;
+ return !Constraints.Pragma &&
+ !Constraints.Type &&
+ !Constraints.Function;
}
};
diff --git a/yql/essentials/sql/v1/complete/name/static/frequency.cpp b/yql/essentials/sql/v1/complete/name/static/frequency.cpp
index d9c8ba9652c..2de082e287e 100644
--- a/yql/essentials/sql/v1/complete/name/static/frequency.cpp
+++ b/yql/essentials/sql/v1/complete/name/static/frequency.cpp
@@ -14,6 +14,7 @@ namespace NSQLComplete {
const char* Sum = "sum";
} Key;
struct {
+ const char* Pragma = "PRAGMA";
const char* Type = "TYPE";
const char* Func = "FUNC";
const char* Module = "MODULE";
@@ -53,14 +54,17 @@ namespace NSQLComplete {
TFrequencyData Convert(TVector<TFrequencyItem> items) {
TFrequencyData data;
for (auto& item : items) {
- if (item.Parent == Json.Parent.Type ||
+ if (item.Parent == Json.Parent.Pragma ||
+ item.Parent == Json.Parent.Type ||
item.Parent == Json.Parent.Func ||
item.Parent == Json.Parent.ModuleFunc ||
item.Parent == Json.Parent.Module) {
item.Rule = ToLowerUTF8(item.Rule);
}
- if (item.Parent == Json.Parent.Type) {
+ if (item.Parent == Json.Parent.Pragma) {
+ data.Pragmas[item.Rule] += item.Sum;
+ } else if (item.Parent == Json.Parent.Type) {
data.Types[item.Rule] += item.Sum;
} else if (item.Parent == Json.Parent.Func ||
item.Parent == Json.Parent.ModuleFunc) {
diff --git a/yql/essentials/sql/v1/complete/name/static/frequency.h b/yql/essentials/sql/v1/complete/name/static/frequency.h
index 3d128f824b4..067453bc404 100644
--- a/yql/essentials/sql/v1/complete/name/static/frequency.h
+++ b/yql/essentials/sql/v1/complete/name/static/frequency.h
@@ -6,6 +6,7 @@
namespace NSQLComplete {
struct TFrequencyData {
+ THashMap<TString, size_t> Pragmas;
THashMap<TString, size_t> Types;
THashMap<TString, size_t> Functions;
};
diff --git a/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp b/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp
index 29c303b3102..9fdf314fee6 100644
--- a/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp
+++ b/yql/essentials/sql/v1/complete/name/static/json_name_set.cpp
@@ -26,6 +26,10 @@ namespace NSQLComplete {
return keys;
}
+ TVector<TString> ParsePragmas(NJson::TJsonValue json) {
+ return ParseNames(json.GetArraySafe());
+ }
+
TVector<TString> ParseTypes(NJson::TJsonValue json) {
return ParseNames(json.GetArraySafe());
}
@@ -48,6 +52,7 @@ namespace NSQLComplete {
NameSet MakeDefaultNameSet() {
return {
+ .Pragmas = ParsePragmas(LoadJsonResource("pragmas_opensource.json")),
.Types = ParseTypes(LoadJsonResource("types.json")),
.Functions = Merge(
ParseFunctions(LoadJsonResource("sql_functions.json")),
diff --git a/yql/essentials/sql/v1/complete/name/static/name_service.cpp b/yql/essentials/sql/v1/complete/name/static/name_service.cpp
index fdb1dd4eae1..42769198579 100644
--- a/yql/essentials/sql/v1/complete/name/static/name_service.cpp
+++ b/yql/essentials/sql/v1/complete/name/static/name_service.cpp
@@ -35,12 +35,43 @@ namespace NSQLComplete {
}
}
+ TString Prefixed(const TStringBuf requestPrefix, const TStringBuf delimeter, const TNamespaced& namespaced) {
+ TString prefix;
+ if (!namespaced.Namespace.empty()) {
+ prefix += namespaced.Namespace;
+ prefix += delimeter;
+ }
+ prefix += requestPrefix;
+ return prefix;
+ }
+
+ void FixPrefix(TString& name, const TStringBuf delimeter, const TNamespaced& namespaced) {
+ if (namespaced.Namespace.empty()) {
+ return;
+ }
+ name.remove(0, namespaced.Namespace.size() + delimeter.size());
+ }
+
+ void FixPrefix(TGenericName& name, const TNameRequest& request) {
+ std::visit([&](auto& name) -> size_t {
+ using T = std::decay_t<decltype(name)>;
+ if constexpr (std::is_same_v<T, TPragmaName>) {
+ FixPrefix(name.Indentifier, ".", *request.Constraints.Pragma);
+ }
+ if constexpr (std::is_same_v<T, TFunctionName>) {
+ FixPrefix(name.Indentifier, "::", *request.Constraints.Function);
+ }
+ return 0;
+ }, name);
+ }
+
class TStaticNameService: public INameService {
public:
explicit TStaticNameService(NameSet names, IRanking::TPtr ranking)
: NameSet_(std::move(names))
, Ranking_(std::move(ranking))
{
+ Sort(NameSet_.Pragmas, NoCaseCompare);
Sort(NameSet_.Types, NoCaseCompare);
Sort(NameSet_.Functions, NoCaseCompare);
}
@@ -48,20 +79,30 @@ namespace NSQLComplete {
TFuture<TNameResponse> Lookup(TNameRequest request) override {
TNameResponse response;
- if (request.Constraints.TypeName) {
+ if (request.Constraints.Pragma) {
+ auto prefix = Prefixed(request.Prefix, ".", *request.Constraints.Pragma);
+ auto names = FilteredByPrefix(prefix, NameSet_.Pragmas);
+ AppendAs<TPragmaName>(response.RankedNames, names);
+ }
+
+ if (request.Constraints.Type) {
AppendAs<TTypeName>(
response.RankedNames,
FilteredByPrefix(request.Prefix, NameSet_.Types));
}
if (request.Constraints.Function) {
- AppendAs<TFunctionName>(
- response.RankedNames,
- FilteredByPrefix(request.Prefix, NameSet_.Functions));
+ auto prefix = Prefixed(request.Prefix, "::", *request.Constraints.Function);
+ auto names = FilteredByPrefix(prefix, NameSet_.Functions);
+ AppendAs<TFunctionName>(response.RankedNames, names);
}
Ranking_->CropToSortedPrefix(response.RankedNames, request.Limit);
+ for (auto& name : response.RankedNames) {
+ FixPrefix(name, request);
+ }
+
return NThreading::MakeFuture(std::move(response));
}
diff --git a/yql/essentials/sql/v1/complete/name/static/name_service.h b/yql/essentials/sql/v1/complete/name/static/name_service.h
index a5c90465c83..348d6ece454 100644
--- a/yql/essentials/sql/v1/complete/name/static/name_service.h
+++ b/yql/essentials/sql/v1/complete/name/static/name_service.h
@@ -7,6 +7,7 @@
namespace NSQLComplete {
struct NameSet {
+ TVector<TString> Pragmas;
TVector<TString> Types;
TVector<TString> Functions;
};
diff --git a/yql/essentials/sql/v1/complete/name/static/ranking.cpp b/yql/essentials/sql/v1/complete/name/static/ranking.cpp
index 45e6e2b2fa2..b3d5c3c8c31 100644
--- a/yql/essentials/sql/v1/complete/name/static/ranking.cpp
+++ b/yql/essentials/sql/v1/complete/name/static/ranking.cpp
@@ -59,6 +59,12 @@ namespace NSQLComplete {
auto identifier = ToLowerUTF8(ContentView(name));
+ if constexpr (std::is_same_v<T, TPragmaName>) {
+ if (auto weight = Frequency_.Pragmas.FindPtr(identifier)) {
+ return *weight;
+ }
+ }
+
if constexpr (std::is_same_v<T, TFunctionName>) {
if (auto weight = Frequency_.Functions.FindPtr(identifier)) {
return *weight;
diff --git a/yql/essentials/sql/v1/complete/name/static/ya.make b/yql/essentials/sql/v1/complete/name/static/ya.make
index 639371447af..3c6547f3b52 100644
--- a/yql/essentials/sql/v1/complete/name/static/ya.make
+++ b/yql/essentials/sql/v1/complete/name/static/ya.make
@@ -13,6 +13,7 @@ PEERDIR(
)
RESOURCE(
+ yql/essentials/data/language/pragmas_opensource.json pragmas_opensource.json
yql/essentials/data/language/types.json types.json
yql/essentials/data/language/sql_functions.json sql_functions.json
yql/essentials/data/language/udfs_basic.json udfs_basic.json
diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp
index ed3afa29df4..85fcf87afd1 100644
--- a/yql/essentials/sql/v1/complete/sql_complete.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete.cpp
@@ -35,10 +35,10 @@ namespace NSQLComplete {
<< " for input size " << input.Text.size();
}
- auto prefix = input.Text.Head(input.CursorPosition);
- auto completedToken = GetCompletedToken(prefix);
+ TLocalSyntaxContext context = SyntaxAnalysis->Analyze(input);
- auto context = SyntaxAnalysis->Analyze(input);
+ TStringBuf prefix = input.Text.Head(input.CursorPosition);
+ TCompletedToken completedToken = GetCompletedToken(prefix);
TVector<TCandidate> candidates;
EnrichWithKeywords(candidates, std::move(context.Keywords), completedToken);
@@ -85,12 +85,20 @@ namespace NSQLComplete {
.Limit = Configuration.Limit - candidates.size(),
};
+ if (context.Pragma) {
+ TPragmaName::TConstraints constraints;
+ constraints.Namespace = context.Pragma->Namespace;
+ request.Constraints.Pragma = std::move(constraints);
+ }
+
if (context.IsTypeName) {
- request.Constraints.TypeName = TTypeName::TConstraints();
+ request.Constraints.Type = TTypeName::TConstraints();
}
- if (context.IsFunctionName) {
- request.Constraints.Function = TFunctionName::TConstraints();
+ if (context.Function) {
+ TFunctionName::TConstraints constraints;
+ constraints.Namespace = context.Function->Namespace;
+ request.Constraints.Function = std::move(constraints);
}
if (request.IsEmpty()) {
@@ -107,6 +115,9 @@ namespace NSQLComplete {
for (auto& name : names) {
candidates.emplace_back(std::visit([](auto&& name) -> TCandidate {
using T = std::decay_t<decltype(name)>;
+ if constexpr (std::is_base_of_v<TPragmaName, T>) {
+ return {ECandidateKind::PragmaName, std::move(name.Indentifier)};
+ }
if constexpr (std::is_base_of_v<TTypeName, T>) {
return {ECandidateKind::TypeName, std::move(name.Indentifier)};
}
@@ -162,6 +173,9 @@ void Out<NSQLComplete::ECandidateKind>(IOutputStream& out, NSQLComplete::ECandid
case NSQLComplete::ECandidateKind::Keyword:
out << "Keyword";
break;
+ case NSQLComplete::ECandidateKind::PragmaName:
+ out << "PragmaName";
+ break;
case NSQLComplete::ECandidateKind::TypeName:
out << "TypeName";
break;
diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h
index b8a970efd8f..5d0271b4dca 100644
--- a/yql/essentials/sql/v1/complete/sql_complete.h
+++ b/yql/essentials/sql/v1/complete/sql_complete.h
@@ -20,6 +20,7 @@ namespace NSQLComplete {
enum class ECandidateKind {
Keyword,
+ PragmaName,
TypeName,
FunctionName,
};
diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
index e9f5dbdfb73..7d595842afb 100644
--- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
+++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp
@@ -39,6 +39,7 @@ public:
Y_UNIT_TEST_SUITE(SqlCompleteTests) {
using ECandidateKind::FunctionName;
using ECandidateKind::Keyword;
+ using ECandidateKind::PragmaName;
using ECandidateKind::TypeName;
TLexerSupplier MakePureLexerSupplier() {
@@ -55,8 +56,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
ISqlCompletionEngine::TPtr MakeSqlCompletionEngineUT() {
TLexerSupplier lexer = MakePureLexerSupplier();
NameSet names = {
+ .Pragmas = {"yson.CastToString"},
.Types = {"Uint64"},
- .Functions = {"StartsWith"},
+ .Functions = {"StartsWith", "DateTime::Split"},
};
auto ranking = MakeDefaultRanking({});
INameService::TPtr service = MakeStaticNameService(std::move(names), std::move(ranking));
@@ -267,12 +269,36 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
}
Y_UNIT_TEST(Pragma) {
- TVector<TCandidate> expected = {
- {Keyword, "ANSI"},
- };
-
auto engine = MakeSqlCompletionEngineUT();
- UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"PRAGMA "}), expected);
+ {
+ TVector<TCandidate> expected = {
+ {Keyword, "ANSI"},
+ {PragmaName, "yson.CastToString"}};
+ auto completion = engine->Complete({"PRAGMA "});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "");
+ }
+ {
+ TVector<TCandidate> expected = {
+ {PragmaName, "yson.CastToString"}};
+ auto completion = engine->Complete({"PRAGMA yson"});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "yson");
+ }
+ {
+ TVector<TCandidate> expected = {
+ {PragmaName, "CastToString"}};
+ auto completion = engine->Complete({"PRAGMA yson."});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "");
+ }
+ {
+ TVector<TCandidate> expected = {
+ {PragmaName, "CastToString"}};
+ auto completion = engine->Complete({"PRAGMA yson.cast"});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "cast");
+ }
}
Y_UNIT_TEST(Select) {
@@ -307,6 +333,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
{Keyword, "TRUE"},
{Keyword, "TUPLE"},
{Keyword, "VARIANT"},
+ {FunctionName, "DateTime::Split("},
{FunctionName, "StartsWith("},
};
@@ -344,6 +371,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
{Keyword, "TRUE"},
{Keyword, "TUPLE"},
{Keyword, "VARIANT"},
+ {FunctionName, "DateTime::Split("},
{FunctionName, "StartsWith("},
};
@@ -402,6 +430,41 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
}
}
+ Y_UNIT_TEST(FunctionName) {
+ auto engine = MakeSqlCompletionEngineUT();
+ {
+ TVector<TCandidate> expected = {
+ {FunctionName, "DateTime::Split("},
+ };
+ auto completion = engine->Complete({"SELECT Date"});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "Date");
+ }
+ {
+ TVector<TCandidate> expected = {
+ {FunctionName, "Split("},
+ };
+ auto completion = engine->Complete({"SELECT DateTime:"});
+ UNIT_ASSERT(completion.Candidates.empty());
+ }
+ {
+ TVector<TCandidate> expected = {
+ {FunctionName, "Split("},
+ };
+ auto completion = engine->Complete({"SELECT DateTime::"});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "");
+ }
+ {
+ TVector<TCandidate> expected = {
+ {FunctionName, "Split("},
+ };
+ auto completion = engine->Complete({"SELECT DateTime::s"});
+ UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);
+ UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "s");
+ }
+ }
+
Y_UNIT_TEST(UTF8Wide) {
auto engine = MakeSqlCompletionEngineUT();
UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"\xF0\x9F\x98\x8A"}).size(), 0);
@@ -410,9 +473,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
Y_UNIT_TEST(WordBreak) {
auto engine = MakeSqlCompletionEngineUT();
- UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT ("}).size(), 29);
- UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT (1)"}).size(), 30);
- UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT 1;"}).size(), 35);
+ UNIT_ASSERT_GE(Complete(engine, {"SELECT ("}).size(), 29);
+ UNIT_ASSERT_GE(Complete(engine, {"SELECT (1)"}).size(), 30);
+ UNIT_ASSERT_GE(Complete(engine, {"SELECT 1;"}).size(), 35);
}
Y_UNIT_TEST(Typing) {
@@ -481,6 +544,16 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
};
UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"SELECT OPTIONAL<U"}), expected);
}
+ {
+ TVector<TCandidate> expected = {
+ {PragmaName, "yson.DisableStrict"},
+ {PragmaName, "yson.AutoConvert"},
+ {PragmaName, "yson.Strict"},
+ {PragmaName, "yson.CastToString"},
+ {PragmaName, "yson.DisableCastToString"},
+ };
+ UNIT_ASSERT_VALUES_EQUAL(Complete(engine, {"PRAGMA yson"}), expected);
+ }
}
Y_UNIT_TEST(OnFailingNameService) {
@@ -516,6 +589,10 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
Y_UNIT_TEST(Ranking) {
TFrequencyData frequency = {
+ .Pragmas = {
+ {"yt.defaultmemorylimit", 16},
+ {"yt.annotations", 8},
+ },
.Types = {
{"int32", 128},
{"int64", 64},
@@ -533,6 +610,15 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {
auto service = MakeStaticNameService(MakeDefaultNameSet(), MakeDefaultRanking(frequency));
auto engine = MakeSqlCompletionEngine(MakePureLexerSupplier(), std::move(service));
{
+ TVector<TCandidate> expectedPrefix = {
+ {PragmaName, "DefaultMemoryLimit"},
+ {PragmaName, "Annotations"},
+ };
+ auto actualPrefix = Complete(engine, {"PRAGMA yt."});
+ actualPrefix.crop(expectedPrefix.size());
+ UNIT_ASSERT_VALUES_EQUAL(actualPrefix, expectedPrefix);
+ }
+ {
TVector<TCandidate> expected = {
{TypeName, "Int32"},
{TypeName, "Int64"},
diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp
index cac43e5a320..28ace474f8e 100644
--- a/yql/essentials/sql/v1/complete/syntax/local.cpp
+++ b/yql/essentials/sql/v1/complete/syntax/local.cpp
@@ -53,10 +53,14 @@ namespace NSQLComplete {
}
auto candidates = C3.Complete(prefix);
+
+ NSQLTranslation::TParsedTokenList tokens = Tokenized(prefix);
+
return {
.Keywords = SiftedKeywords(candidates),
+ .Pragma = PragmaMatch(tokens, candidates),
.IsTypeName = IsTypeNameMatched(candidates),
- .IsFunctionName = IsFunctionNameMatched(candidates),
+ .Function = FunctionMatch(tokens, candidates),
};
}
@@ -131,16 +135,73 @@ namespace NSQLComplete {
return name;
}
+ std::optional<TLocalSyntaxContext::TPragma> PragmaMatch(
+ const NSQLTranslation::TParsedTokenList& tokens, const TC3Candidates& candidates) {
+ bool isMatched = AnyOf(candidates.Rules, [&](const TMatchedRule& rule) {
+ return IsLikelyPragmaStack(rule.ParserCallStack);
+ });
+ if (!isMatched) {
+ return std::nullopt;
+ }
+
+ TLocalSyntaxContext::TPragma pragma;
+ if (EndsWith(tokens, {"ID_PLAIN", "DOT"})) {
+ pragma.Namespace = tokens[tokens.size() - 2].Content;
+ } else if (EndsWith(tokens, {"ID_PLAIN", "DOT", ""})) {
+ pragma.Namespace = tokens[tokens.size() - 3].Content;
+ }
+ return pragma;
+ }
+
bool IsTypeNameMatched(const TC3Candidates& candidates) {
return AnyOf(candidates.Rules, [&](const TMatchedRule& rule) {
return IsLikelyTypeStack(rule.ParserCallStack);
});
}
- bool IsFunctionNameMatched(const TC3Candidates& candidates) {
- return AnyOf(candidates.Rules, [&](const TMatchedRule& rule) {
+ std::optional<TLocalSyntaxContext::TFunction> FunctionMatch(
+ const NSQLTranslation::TParsedTokenList& tokens, const TC3Candidates& candidates) {
+ bool isMatched = AnyOf(candidates.Rules, [&](const TMatchedRule& rule) {
return IsLikelyFunctionStack(rule.ParserCallStack);
});
+ if (!isMatched) {
+ return std::nullopt;
+ }
+
+ TLocalSyntaxContext::TFunction function;
+ if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE"})) {
+ function.Namespace = tokens[tokens.size() - 2].Content;
+ } else if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE", ""})) {
+ function.Namespace = tokens[tokens.size() - 3].Content;
+ }
+ return function;
+ }
+
+ NSQLTranslation::TParsedTokenList Tokenized(const TStringBuf text) {
+ NSQLTranslation::TParsedTokenList tokens;
+ NYql::TIssues issues;
+ if (!NSQLTranslation::Tokenize(
+ *Lexer_, TString(text), /* queryName = */ "",
+ tokens, issues, /* maxErrors = */ 0)) {
+ return {};
+ }
+ Y_ENSURE(!tokens.empty() && tokens.back().Name == "EOF");
+ tokens.pop_back();
+ return tokens;
+ }
+
+ bool EndsWith(
+ const NSQLTranslation::TParsedTokenList& tokens,
+ const TVector<TStringBuf>& pattern) {
+ if (tokens.size() < pattern.size()) {
+ return false;
+ }
+ for (yssize_t i = tokens.ysize() - 1, j = pattern.ysize() - 1; 0 <= j; --i, --j) {
+ if (!pattern[j].empty() && tokens[i].Name != pattern[j]) {
+ return false;
+ }
+ }
+ return true;
}
const ISqlGrammar* Grammar;
diff --git a/yql/essentials/sql/v1/complete/syntax/local.h b/yql/essentials/sql/v1/complete/syntax/local.h
index 79984d00e2b..24e78108e10 100644
--- a/yql/essentials/sql/v1/complete/syntax/local.h
+++ b/yql/essentials/sql/v1/complete/syntax/local.h
@@ -10,9 +10,18 @@
namespace NSQLComplete {
struct TLocalSyntaxContext {
+ struct TPragma {
+ TString Namespace;
+ };
+
+ struct TFunction {
+ TString Namespace;
+ };
+
TVector<TString> Keywords;
+ std::optional<TPragma> Pragma;
bool IsTypeName;
- bool IsFunctionName;
+ std::optional<TFunction> Function;
};
class ILocalSyntaxAnalysis {
diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp
index 57e058fa900..1bfcac47266 100644
--- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp
+++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp
@@ -24,6 +24,11 @@ namespace NSQLComplete {
RULE(Keyword_compat),
};
+ const TVector<TRuleId> PragmaNameRules = {
+ RULE(Opt_id_prefix_or_type),
+ RULE(An_id),
+ };
+
const TVector<TRuleId> TypeNameRules = {
RULE(Type_name_simple),
RULE(An_id_or_type),
@@ -62,6 +67,11 @@ namespace NSQLComplete {
return Find(stack, rule) != std::end(stack);
}
+ bool IsLikelyPragmaStack(const TParserCallStack& stack) {
+ return EndsWith({RULE(Pragma_stmt), RULE(Opt_id_prefix_or_type)}, stack) ||
+ EndsWith({RULE(Pragma_stmt), RULE(An_id)}, stack);
+ }
+
bool IsLikelyTypeStack(const TParserCallStack& stack) {
return EndsWith({RULE(Type_name_simple)}, stack) ||
(Contains({RULE(Invoke_expr),
@@ -75,12 +85,14 @@ namespace NSQLComplete {
return EndsWith({RULE(Unary_casual_subexpr), RULE(Id_expr)}, stack) ||
EndsWith({RULE(Unary_casual_subexpr),
RULE(Atom_expr),
- RULE(An_id_or_type)}, stack);
+ RULE(An_id_or_type)}, stack) ||
+ EndsWith({RULE(Atom_expr), RULE(Id_or_type)}, stack);
}
std::unordered_set<TRuleId> GetC3PreferredRules() {
std::unordered_set<TRuleId> preferredRules;
preferredRules.insert(std::begin(KeywordRules), std::end(KeywordRules));
+ preferredRules.insert(std::begin(PragmaNameRules), std::end(PragmaNameRules));
preferredRules.insert(std::begin(TypeNameRules), std::end(TypeNameRules));
preferredRules.insert(std::begin(FunctionNameRules), std::end(FunctionNameRules));
return preferredRules;
diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h
index 756586988db..94533bddaae 100644
--- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h
+++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h
@@ -4,6 +4,8 @@
namespace NSQLComplete {
+ bool IsLikelyPragmaStack(const TParserCallStack& stack);
+
bool IsLikelyTypeStack(const TParserCallStack& stack);
bool IsLikelyFunctionStack(const TParserCallStack& stack);