diff options
author | vityaman <[email protected]> | 2025-05-06 15:49:02 +0300 |
---|---|---|
committer | robot-piglet <[email protected]> | 2025-05-06 16:04:08 +0300 |
commit | 9c3fdca51d8ae892c5ad8f6ef92df73fafc09e28 (patch) | |
tree | 561c599fae4ea29b537a6958b65e1b052548edf2 | |
parent | c131e959456f9f9a4adada5623ce3bae4097a8c1 (diff) |
YQL-19747 Complete folder, table and cluster names
---
- Related to `YQL-19747`
- On top of https://github.com/ytsaurus/ytsaurus/pull/1253
- Related to https://github.com/ydb-platform/ydb/issues/9056
- Related to https://github.com/vityaman/ydb/issues/14
- Related to https://github.com/vityaman/ydb/issues/35
- Related to https://github.com/vityaman/ydb/issues/40
---
Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1257
commit_hash:0b842abb27184c88b8177beeea29fb1ea86b7a04
58 files changed, 1541 insertions, 395 deletions
diff --git a/yql/essentials/sql/v1/complete/antlr4/c3t.h b/yql/essentials/sql/v1/complete/antlr4/c3t.h index 35b1f714fa7..aca5ebf92e5 100644 --- a/yql/essentials/sql/v1/complete/antlr4/c3t.h +++ b/yql/essentials/sql/v1/complete/antlr4/c3t.h @@ -27,37 +27,37 @@ namespace NSQLComplete { class TC3Engine: public IC3Engine { public: explicit TC3Engine(TConfig config) - : Chars() - , Lexer(&Chars) - , Tokens(&Lexer) - , Parser(&Tokens) - , CompletionCore(&Parser) + : Chars_() + , Lexer_(&Chars_) + , Tokens_(&Lexer_) + , Parser_(&Tokens_) + , CompletionCore_(&Parser_) { - Lexer.removeErrorListeners(); - Parser.removeErrorListeners(); + Lexer_.removeErrorListeners(); + Parser_.removeErrorListeners(); - CompletionCore.ignoredTokens = std::move(config.IgnoredTokens); - CompletionCore.preferredRules = std::move(config.PreferredRules); + CompletionCore_.ignoredTokens = std::move(config.IgnoredTokens); + CompletionCore_.preferredRules = std::move(config.PreferredRules); } TC3Candidates Complete(TCompletionInput input) override { auto prefix = input.Text.Head(input.CursorPosition); Assign(prefix); const auto caretTokenIndex = CaretTokenIndex(prefix); - auto candidates = CompletionCore.collectCandidates(caretTokenIndex); + auto candidates = CompletionCore_.collectCandidates(caretTokenIndex); return Converted(std::move(candidates)); } private: void Assign(TStringBuf prefix) { - Chars.load(prefix.Data(), prefix.Size(), /* lenient = */ false); - Lexer.reset(); - Tokens.setTokenSource(&Lexer); - Tokens.fill(); + Chars_.load(prefix.Data(), prefix.Size(), /* lenient = */ false); + Lexer_.reset(); + Tokens_.setTokenSource(&Lexer_); + Tokens_.fill(); } size_t CaretTokenIndex(TStringBuf prefix) { - const auto tokensCount = Tokens.size(); + const auto tokensCount = Tokens_.size(); if (2 <= tokensCount && !LastWord(prefix).Empty()) { return tokensCount - 2; } @@ -76,11 +76,11 @@ namespace NSQLComplete { return converted; } - antlr4::ANTLRInputStream Chars; - G::TLexer Lexer; - antlr4::BufferedTokenStream Tokens; - G::TParser Parser; - c3::CodeCompletionCore CompletionCore; + antlr4::ANTLRInputStream Chars_; + G::TLexer Lexer_; + antlr4::BufferedTokenStream Tokens_; + G::TParser Parser_; + c3::CodeCompletionCore CompletionCore_; }; } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/input.cpp b/yql/essentials/sql/v1/complete/core/input.cpp new file mode 100644 index 00000000000..8eca3a28ee8 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/input.cpp @@ -0,0 +1,25 @@ +#include "input.h" + +#include <util/generic/yexception.h> + +namespace NSQLComplete { + + TCompletionInput SharpedInput(TString& text) { + constexpr char delim = '#'; + + size_t pos = text.find_first_of(delim); + if (pos == TString::npos) { + return { + .Text = text, + }; + } + + Y_ENSURE(!TStringBuf(text).Tail(pos + 1).Contains(delim)); + text.erase(std::begin(text) + pos); + return { + .Text = text, + .CursorPosition = pos, + }; + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/input.h b/yql/essentials/sql/v1/complete/core/input.h index 3bb609cbb22..d852736cb44 100644 --- a/yql/essentials/sql/v1/complete/core/input.h +++ b/yql/essentials/sql/v1/complete/core/input.h @@ -9,4 +9,6 @@ namespace NSQLComplete { size_t CursorPosition = Text.length(); }; + TCompletionInput SharpedInput(TString& text Y_LIFETIME_BOUND); + } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/name.h b/yql/essentials/sql/v1/complete/core/name.h new file mode 100644 index 00000000000..02524766de1 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/name.h @@ -0,0 +1,10 @@ +#pragma once + +namespace NSQLComplete { + + enum class EObjectKind { + Folder, + Table, + }; + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/ya.make b/yql/essentials/sql/v1/complete/core/ya.make index 9865d255c8f..8bc457f8f95 100644 --- a/yql/essentials/sql/v1/complete/core/ya.make +++ b/yql/essentials/sql/v1/complete/core/ya.make @@ -1,3 +1,7 @@ LIBRARY() +SRCS( + input.cpp +) + END() diff --git a/yql/essentials/sql/v1/complete/name/cluster/discovery.h b/yql/essentials/sql/v1/complete/name/cluster/discovery.h new file mode 100644 index 00000000000..6b496f15546 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/discovery.h @@ -0,0 +1,19 @@ +#pragma once + +#include <library/cpp/threading/future/core/future.h> + +#include <util/generic/ptr.h> + +namespace NSQLComplete { + + using TClusterList = TVector<TString>; + + class IClusterDiscovery: public TThrRefBase { + public: + using TPtr = TIntrusivePtr<IClusterDiscovery>; + + virtual ~IClusterDiscovery() = default; + virtual NThreading::TFuture<TClusterList> Query() const = 0; + }; + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/cluster/static/discovery.cpp b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.cpp new file mode 100644 index 00000000000..7caee64c182 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.cpp @@ -0,0 +1,28 @@ +#include "discovery.h" + +namespace NSQLComplete { + + namespace { + + class TClusterDiscovery: public IClusterDiscovery { + public: + explicit TClusterDiscovery(TVector<TString> instances) + : ClusterList_(std::move(instances)) + { + } + + NThreading::TFuture<TClusterList> Query() const override { + return NThreading::MakeFuture(ClusterList_); + } + + private: + TVector<TString> ClusterList_; + }; + + } // namespace + + IClusterDiscovery::TPtr MakeStaticClusterDiscovery(TVector<TString> instances) { + return new TClusterDiscovery(std::move(instances)); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/cluster/static/discovery.h b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.h new file mode 100644 index 00000000000..bfad0eed62f --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/cluster/discovery.h> + +namespace NSQLComplete { + + IClusterDiscovery::TPtr MakeStaticClusterDiscovery(TVector<TString> instances); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/cluster/static/ya.make b/yql/essentials/sql/v1/complete/name/cluster/static/ya.make new file mode 100644 index 00000000000..567130a2ff0 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/static/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + discovery.cpp +) + +PEERDIR( + yql/essentials/sql/v1/complete/name/cluster +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/cluster/ya.make b/yql/essentials/sql/v1/complete/name/cluster/ya.make new file mode 100644 index 00000000000..5ea880aeadd --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +PEERDIR( + library/cpp/threading/future +) + +END() + +RECURSE( + static +) diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp new file mode 100644 index 00000000000..f6d79b280a0 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp @@ -0,0 +1,36 @@ +#include "schema.h" + +namespace NSQLComplete { + + namespace { + + class TSchema: public ISchema { + public: + explicit TSchema(THashMap<TString, ISchema::TPtr> mapping) + : Mapping_(std::move(mapping)) + { + } + + NThreading::TFuture<TListResponse> List(const TListRequest& request) const override { + auto iter = Mapping_.find(request.Cluster); + if (iter == std::end(Mapping_)) { + yexception e; + e << "unknown cluster '" << request.Cluster << "'"; + std::exception_ptr p = std::make_exception_ptr(e); + return NThreading::MakeErrorFuture<TListResponse>(p); + } + + return iter->second->List(request); + } + + private: + THashMap<TString, ISchema::TPtr> Mapping_; + }; + + } // namespace + + ISchema::TPtr MakeDispatchSchema(THashMap<TString, ISchema::TPtr> mapping) { + return new TSchema(std::move(mapping)); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h new file mode 100644 index 00000000000..517a3ad0af7 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/object/schema.h> + +namespace NSQLComplete { + + ISchema::TPtr MakeDispatchSchema(THashMap<TString, ISchema::TPtr> mapping); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make b/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make new file mode 100644 index 00000000000..071bf5dff7d --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + schema.cpp +) + +PEERDIR( + yql/essentials/sql/v1/complete/name/object +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/object/schema_gateway.cpp b/yql/essentials/sql/v1/complete/name/object/schema.cpp index c802ddcb7a1..ba9e09b2ee7 100644 --- a/yql/essentials/sql/v1/complete/name/object/schema_gateway.cpp +++ b/yql/essentials/sql/v1/complete/name/object/schema.cpp @@ -1,4 +1,4 @@ -#include "schema_gateway.h" +#include "schema.h" template <> void Out<NSQLComplete::TFolderEntry>(IOutputStream& out, const NSQLComplete::TFolderEntry& entry) { diff --git a/yql/essentials/sql/v1/complete/name/object/schema_gateway.h b/yql/essentials/sql/v1/complete/name/object/schema.h index f9307bf495d..687f92d7e8a 100644 --- a/yql/essentials/sql/v1/complete/name/object/schema_gateway.h +++ b/yql/essentials/sql/v1/complete/name/object/schema.h @@ -41,11 +41,11 @@ namespace NSQLComplete { TVector<TFolderEntry> Entries; }; - class ISchemaGateway: public TThrRefBase { + class ISchema: public TThrRefBase { public: - using TPtr = TIntrusivePtr<ISchemaGateway>; + using TPtr = TIntrusivePtr<ISchema>; - virtual ~ISchemaGateway() = default; + virtual ~ISchema() = default; virtual NThreading::TFuture<TListResponse> List(const TListRequest& request) const = 0; }; diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.cpp b/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp index e8e7bf3ccd9..c7b62946f64 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp @@ -1,4 +1,4 @@ -#include "schema_gateway.h" +#include "schema.h" #include <util/charset/utf8.h> @@ -6,7 +6,7 @@ namespace NSQLComplete { namespace { - class TSimpleSchemaGateway: public ISchemaGateway { + class TSimpleSchema: public ISchema { private: static auto FilterByName(TString name) { return [name = std::move(name)](auto f) { @@ -47,14 +47,20 @@ namespace NSQLComplete { } public: - explicit TSimpleSchemaGateway(ISimpleSchemaGateway::TPtr simple) + explicit TSimpleSchema(ISimpleSchema::TPtr simple) : Simple_(std::move(simple)) { } NThreading::TFuture<TListResponse> List(const TListRequest& request) const override { auto [path, name] = Simple_->Split(request.Path); - return Simple_->List(TString(path)) + + TString pathStr(path); + if (!pathStr.StartsWith('/')) { + pathStr.prepend('/'); + } + + return Simple_->List(std::move(pathStr)) .Apply(FilterByName(TString(name))) .Apply(FilterByTypes(std::move(request.Filter.Types))) .Apply(Crop(request.Limit)) @@ -62,13 +68,13 @@ namespace NSQLComplete { } private: - ISimpleSchemaGateway::TPtr Simple_; + ISimpleSchema::TPtr Simple_; }; } // namespace - ISchemaGateway::TPtr MakeSimpleSchemaGateway(ISimpleSchemaGateway::TPtr simple) { - return ISchemaGateway::TPtr(new TSimpleSchemaGateway(std::move(simple))); + ISchema::TPtr MakeSimpleSchema(ISimpleSchema::TPtr simple) { + return ISchema::TPtr(new TSimpleSchema(std::move(simple))); } } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.h b/yql/essentials/sql/v1/complete/name/object/simple/schema.h index 4b4671f1cca..67def573a73 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema.h @@ -1,6 +1,6 @@ #pragma once -#include <yql/essentials/sql/v1/complete/name/object/schema_gateway.h> +#include <yql/essentials/sql/v1/complete/name/object/schema.h> namespace NSQLComplete { @@ -9,15 +9,15 @@ namespace NSQLComplete { TStringBuf NameHint; }; - class ISimpleSchemaGateway: public TThrRefBase { + class ISimpleSchema: public TThrRefBase { public: - using TPtr = TIntrusivePtr<ISimpleSchemaGateway>; + using TPtr = TIntrusivePtr<ISimpleSchema>; - virtual ~ISimpleSchemaGateway() = default; + virtual ~ISimpleSchema() = default; virtual TSplittedPath Split(TStringBuf path) const = 0; virtual NThreading::TFuture<TVector<TFolderEntry>> List(TString folder) const = 0; }; - ISchemaGateway::TPtr MakeSimpleSchemaGateway(ISimpleSchemaGateway::TPtr simple); + ISchema::TPtr MakeSimpleSchema(ISimpleSchema::TPtr simple); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway_ut.cpp b/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp index 86c8118f197..954ecc4da75 100644 --- a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway_ut.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp @@ -1,12 +1,14 @@ -#include "schema_gateway.h" +#include "schema.h" + +#include <yql/essentials/sql/v1/complete/name/object/simple/static/schema.h> #include <library/cpp/testing/unittest/registar.h> using namespace NSQLComplete; -Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { +Y_UNIT_TEST_SUITE(StaticSchemaTests) { - ISchemaGateway::TPtr MakeStaticSchemaGatewayUT() { + ISchema::TPtr MakeStaticSchemaUT() { THashMap<TString, TVector<TFolderEntry>> fs = { {"/", {{"Folder", "local"}, {"Folder", "test"}, @@ -18,18 +20,19 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { {"Table", "meta"}}}, {"/test/service/", {{"Table", "example"}}}, }; - return MakeStaticSchemaGateway(std::move(fs)); + return MakeSimpleSchema( + MakeStaticSimpleSchema(std::move(fs))); } Y_UNIT_TEST(ListFolderBasic) { - auto gateway = MakeStaticSchemaGatewayUT(); + auto schema = MakeStaticSchemaUT(); { TVector<TFolderEntry> expected = { {"Folder", "local"}, {"Folder", "test"}, {"Folder", "prod"}, }; - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/"}).GetValueSync().Entries, expected); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/"}).GetValueSync().Entries, expected); } { TVector<TFolderEntry> expected = { @@ -37,29 +40,29 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { {"Table", "account"}, {"Table", "abacaba"}, }; - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/local/"}).GetValueSync().Entries, expected); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/local/"}).GetValueSync().Entries, expected); } { TVector<TFolderEntry> expected = { {"Folder", "service"}, {"Table", "meta"}, }; - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/test/"}).GetValueSync().Entries, expected); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/test/"}).GetValueSync().Entries, expected); } { TVector<TFolderEntry> expected = { {"Table", "example"}}; - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/test/service/"}).GetValueSync().Entries, expected); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/test/service/"}).GetValueSync().Entries, expected); } } Y_UNIT_TEST(ListFolderHint) { - auto gateway = MakeStaticSchemaGatewayUT(); + auto schema = MakeStaticSchemaUT(); { TVector<TFolderEntry> expected = { {"Folder", "local"}, }; - auto actual = gateway->List({.Path = "/l"}).GetValueSync(); + auto actual = schema->List({.Path = "/l"}).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(actual.Entries, expected); UNIT_ASSERT_VALUES_EQUAL(actual.NameHintLength, 1); } @@ -68,7 +71,7 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { {"Table", "account"}, {"Table", "abacaba"}, }; - auto actual = gateway->List({.Path = "/local/a"}).GetValueSync(); + auto actual = schema->List({.Path = "/local/a"}).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(actual.Entries, expected); UNIT_ASSERT_VALUES_EQUAL(actual.NameHintLength, 1); } @@ -76,14 +79,14 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { TVector<TFolderEntry> expected = { {"Folder", "service"}, }; - auto actual = gateway->List({.Path = "/test/service"}).GetValueSync(); + auto actual = schema->List({.Path = "/test/service"}).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(actual.Entries, expected); UNIT_ASSERT_VALUES_EQUAL(actual.NameHintLength, 7); } } Y_UNIT_TEST(ListFolderFilterByType) { - auto gateway = MakeStaticSchemaGatewayUT(); + auto schema = MakeStaticSchemaUT(); { TVector<TFolderEntry> expected = { {"Folder", "service"}, @@ -94,7 +97,7 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { .Types = THashSet<TString>{"Folder"}, }, }; - UNIT_ASSERT_VALUES_EQUAL(gateway->List(request).GetValueSync().Entries, expected); + UNIT_ASSERT_VALUES_EQUAL(schema->List(request).GetValueSync().Entries, expected); } { TVector<TFolderEntry> expected = { @@ -106,18 +109,18 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { .Types = THashSet<TString>{"Table"}, }, }; - UNIT_ASSERT_VALUES_EQUAL(gateway->List(request).GetValueSync().Entries, expected); + UNIT_ASSERT_VALUES_EQUAL(schema->List(request).GetValueSync().Entries, expected); } } Y_UNIT_TEST(ListFolderLimit) { - auto gateway = MakeStaticSchemaGatewayUT(); - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 0}).GetValueSync().Entries.size(), 0); - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 1}).GetValueSync().Entries.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 2}).GetValueSync().Entries.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 3}).GetValueSync().Entries.size(), 3); - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 4}).GetValueSync().Entries.size(), 3); - UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 5}).GetValueSync().Entries.size(), 3); + auto schema = MakeStaticSchemaUT(); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 0}).GetValueSync().Entries.size(), 0); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 1}).GetValueSync().Entries.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 2}).GetValueSync().Entries.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 3}).GetValueSync().Entries.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 4}).GetValueSync().Entries.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 5}).GetValueSync().Entries.size(), 3); } -} // Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) +} // Y_UNIT_TEST_SUITE(StaticSchemaTests) diff --git a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.cpp b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp index f43af57c752..0af3ff0ef96 100644 --- a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp @@ -1,16 +1,12 @@ -#include "schema_gateway.h" - -#include <yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.h> - -#include <util/charset/utf8.h> +#include "schema.h" namespace NSQLComplete { namespace { - class TSimpleSchemaGateway: public ISimpleSchemaGateway { + class TSimpleSchema: public ISimpleSchema { public: - explicit TSimpleSchemaGateway(THashMap<TString, TVector<TFolderEntry>> data) + explicit TSimpleSchema(THashMap<TString, TVector<TFolderEntry>> data) : Data_(std::move(data)) { for (const auto& [k, _] : Data_) { @@ -44,10 +40,8 @@ namespace NSQLComplete { } // namespace - ISchemaGateway::TPtr MakeStaticSchemaGateway(THashMap<TString, TVector<TFolderEntry>> fs) { - return MakeSimpleSchemaGateway( - ISimpleSchemaGateway::TPtr( - new TSimpleSchemaGateway(std::move(fs)))); + ISimpleSchema::TPtr MakeStaticSimpleSchema(THashMap<TString, TVector<TFolderEntry>> fs) { + return new TSimpleSchema(std::move(fs)); } } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h new file mode 100644 index 00000000000..f04c89f0b23 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/object/simple/schema.h> + +namespace NSQLComplete { + + ISimpleSchema::TPtr MakeStaticSimpleSchema( + THashMap<TString, TVector<TFolderEntry>> fs); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/ya.make b/yql/essentials/sql/v1/complete/name/object/simple/static/ya.make new file mode 100644 index 00000000000..8e7918ed011 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + schema.cpp +) + +PEERDIR( + yql/essentials/sql/v1/complete/name/object/simple +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/object/simple/ut/ya.make b/yql/essentials/sql/v1/complete/name/object/simple/ut/ya.make new file mode 100644 index 00000000000..048dc38d7d8 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/simple/ut/ya.make @@ -0,0 +1,11 @@ +UNITTEST_FOR(yql/essentials/sql/v1/complete/name/object/simple) + +SRCS( + schema_ut.cpp +) + +PEERDIR( + yql/essentials/sql/v1/complete/name/object/simple/static +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/object/simple/ya.make b/yql/essentials/sql/v1/complete/name/object/simple/ya.make index d3668fdb1fc..56eafc1b848 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/ya.make +++ b/yql/essentials/sql/v1/complete/name/object/simple/ya.make @@ -1,7 +1,7 @@ LIBRARY() SRCS( - schema_gateway.cpp + schema.cpp ) PEERDIR( @@ -9,3 +9,11 @@ PEERDIR( ) END() + +RECURSE( + static +) + +RECURSE_FOR_TESTS( + ut +) diff --git a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.h b/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.h deleted file mode 100644 index fd23a956d25..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include <yql/essentials/sql/v1/complete/name/object/schema_gateway.h> - -namespace NSQLComplete { - - ISchemaGateway::TPtr MakeStaticSchemaGateway(THashMap<TString, TVector<TFolderEntry>> fs); - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/static/ut/ya.make b/yql/essentials/sql/v1/complete/name/object/static/ut/ya.make deleted file mode 100644 index 877f015d806..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/static/ut/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -UNITTEST_FOR(yql/essentials/sql/v1/complete/name/object/static) - -SRCS( - schema_gateway_ut.cpp -) - -END() diff --git a/yql/essentials/sql/v1/complete/name/object/static/ya.make b/yql/essentials/sql/v1/complete/name/object/static/ya.make deleted file mode 100644 index d37495f0700..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/static/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -SRCS( - schema_gateway.cpp -) - -PEERDIR( - yql/essentials/sql/v1/complete/name/object - yql/essentials/sql/v1/complete/name/object/simple -) - -END() - -RECURSE_FOR_TESTS( - ut -) diff --git a/yql/essentials/sql/v1/complete/name/object/ya.make b/yql/essentials/sql/v1/complete/name/object/ya.make index 483f11c9a59..2561c018292 100644 --- a/yql/essentials/sql/v1/complete/name/object/ya.make +++ b/yql/essentials/sql/v1/complete/name/object/ya.make @@ -1,7 +1,7 @@ LIBRARY() SRCS( - schema_gateway.cpp + schema.cpp ) PEERDIR( @@ -11,6 +11,6 @@ PEERDIR( END() RECURSE( + dispatch simple - static ) diff --git a/yql/essentials/sql/v1/complete/name/service/cluster/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.cpp new file mode 100644 index 00000000000..db0ba00b667 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.cpp @@ -0,0 +1,82 @@ +#include "name_service.h" + +#include <util/charset/utf8.h> + +namespace NSQLComplete { + + namespace { + + class TNameService: public INameService { + private: + static auto FilterByName(TString name) { + return [name = std::move(name)](auto f) { + TClusterList clusters = f.ExtractValue(); + EraseIf(clusters, [prefix = ToLowerUTF8(name)](const TString& instance) { + return !instance.StartsWith(prefix); + }); + return clusters; + }; + } + + static auto Crop(size_t limit) { + return [limit](auto f) { + TClusterList clusters = f.ExtractValue(); + clusters.crop(limit); + return clusters; + }; + } + + static auto ToResponse(TNameConstraints constraints) { + return [constraints = std::move(constraints)](auto f) { + TClusterList clusters = f.ExtractValue(); + + TNameResponse response; + response.RankedNames.reserve(clusters.size()); + + for (auto& cluster : clusters) { + TClusterName name; + name.Indentifier = std::move(cluster); + response.RankedNames.emplace_back(std::move(name)); + } + + response.RankedNames = constraints.Unqualified(std::move(response.RankedNames)); + return response; + }; + } + + public: + explicit TNameService(IClusterDiscovery::TPtr discovery) + : Discovery_(std::move(discovery)) + { + } + + NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const override { + if (!request.Constraints.Cluster) { + return NThreading::MakeFuture<TNameResponse>({}); + } + + return Discovery_->Query() + .Apply(FilterByName(QualifiedClusterName(request))) + .Apply(Crop(request.Limit)) + .Apply(ToResponse(request.Constraints)); + } + + private: + static TString QualifiedClusterName(const TNameRequest& request) { + TClusterName cluster; + cluster.Indentifier = request.Prefix; + + TGenericName generic = request.Constraints.Qualified(cluster); + return std::get<TClusterName>(std::move(generic)).Indentifier; + } + + IClusterDiscovery::TPtr Discovery_; + }; + + } // namespace + + INameService::TPtr MakeClusterNameService(IClusterDiscovery::TPtr discovery) { + return new TNameService(std::move(discovery)); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/cluster/name_service.h b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.h new file mode 100644 index 00000000000..a57eabc0d2b --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/cluster/discovery.h> +#include <yql/essentials/sql/v1/complete/name/service/name_service.h> + +namespace NSQLComplete { + + INameService::TPtr MakeClusterNameService(IClusterDiscovery::TPtr discovery); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/cluster/ya.make b/yql/essentials/sql/v1/complete/name/service/cluster/ya.make new file mode 100644 index 00000000000..4849690a6f2 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/cluster/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + name_service.cpp +) + +PEERDIR( + yql/essentials/sql/v1/complete/name/cluster + yql/essentials/sql/v1/complete/name/service +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/service/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/name_service.cpp index 88473a60bff..92d8b50e98f 100644 --- a/yql/essentials/sql/v1/complete/name/service/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/name_service.cpp @@ -34,6 +34,8 @@ namespace NSQLComplete { SetPrefix(name.Indentifier, ".", *Pragma); } else if constexpr (std::is_same_v<T, TFunctionName>) { SetPrefix(name.Indentifier, "::", *Function); + } else if constexpr (std::is_same_v<T, TClusterName>) { + SetPrefix(name.Indentifier, ":", *Cluster); } return name; }, std::move(unqualified)); @@ -46,6 +48,8 @@ namespace NSQLComplete { FixPrefix(name.Indentifier, ".", *Pragma); } else if constexpr (std::is_same_v<T, TFunctionName>) { FixPrefix(name.Indentifier, "::", *Function); + } else if constexpr (std::is_same_v<T, TClusterName>) { + FixPrefix(name.Indentifier, ":", *Cluster); } return name; }, std::move(qualified)); diff --git a/yql/essentials/sql/v1/complete/name/service/name_service.h b/yql/essentials/sql/v1/complete/name/service/name_service.h index 7d773582b61..cdcfc3a4982 100644 --- a/yql/essentials/sql/v1/complete/name/service/name_service.h +++ b/yql/essentials/sql/v1/complete/name/service/name_service.h @@ -1,5 +1,6 @@ #pragma once +#include <yql/essentials/sql/v1/complete/core/name.h> #include <yql/essentials/sql/v1/complete/core/statement.h> #include <library/cpp/threading/future/core/future.h> @@ -7,11 +8,10 @@ #include <util/generic/vector.h> #include <util/generic/string.h> #include <util/generic/maybe.h> +#include <util/generic/hash_set.h> namespace NSQLComplete { - using NThreading::TFuture; // TODO(YQL-19747): remove - struct TIndentifier { TString Indentifier; }; @@ -29,7 +29,7 @@ namespace NSQLComplete { }; struct TTypeName: TIndentifier { - using TConstraints = std::monostate; + struct TConstraints {}; }; struct TFunctionName: TIndentifier { @@ -42,18 +42,45 @@ namespace NSQLComplete { }; }; + struct TObjectNameConstraints { + TString Provider; + TString Cluster; + THashSet<EObjectKind> Kinds; + }; + + struct TFolderName: TIndentifier { + }; + + struct TTableName: TIndentifier { + }; + + struct TClusterName: TIndentifier { + struct TConstraints: TNamespaced {}; + }; + + struct TUnkownName { + TString Content; + TString Type; + }; + using TGenericName = std::variant< TKeyword, TPragmaName, TTypeName, TFunctionName, - THintName>; + THintName, + TFolderName, + TTableName, + TClusterName, + TUnkownName>; struct TNameConstraints { TMaybe<TPragmaName::TConstraints> Pragma; TMaybe<TTypeName::TConstraints> Type; TMaybe<TFunctionName::TConstraints> Function; TMaybe<THintName::TConstraints> Hint; + TMaybe<TObjectNameConstraints> Object; + TMaybe<TClusterName::TConstraints> Cluster; TGenericName Qualified(TGenericName unqualified) const; TGenericName Unqualified(TGenericName qualified) const; @@ -72,19 +99,26 @@ namespace NSQLComplete { !Constraints.Pragma && !Constraints.Type && !Constraints.Function && - !Constraints.Hint; + !Constraints.Hint && + !Constraints.Object && + !Constraints.Cluster; } }; struct TNameResponse { TVector<TGenericName> RankedNames; + TMaybe<size_t> NameHintLength; + + bool IsEmpty() const { + return RankedNames.empty(); + } }; class INameService: public TThrRefBase { public: using TPtr = TIntrusivePtr<INameService>; - virtual TFuture<TNameResponse> Lookup(TNameRequest request) const = 0; + virtual NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const = 0; virtual ~INameService() = default; }; diff --git a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp index 3e2dd322522..6b8aa42bc5e 100644 --- a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp +++ b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp @@ -23,7 +23,7 @@ namespace NSQLComplete { TVector<TGenericName>& names, const TNameConstraints& constraints, size_t limit) const override { - limit = std::min(limit, names.size()); + limit = Min(limit, names.size()); TVector<TRow> rows; rows.reserve(names.size()); @@ -91,6 +91,15 @@ namespace NSQLComplete { } } + if constexpr (std::is_same_v<T, TFolderName> || + std::is_same_v<T, TTableName>) { + return std::numeric_limits<size_t>::max(); + } + + if constexpr (std::is_same_v<T, TClusterName>) { + return std::numeric_limits<size_t>::max() - 8; + } + return 0; }, name); } @@ -108,6 +117,9 @@ namespace NSQLComplete { if constexpr (std::is_base_of_v<TIndentifier, T>) { return name.Indentifier; } + if constexpr (std::is_base_of_v<TUnkownName, T>) { + return name.Content; + } }, name); } diff --git a/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp new file mode 100644 index 00000000000..de8e8db65ac --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp @@ -0,0 +1,100 @@ +#include "name_service.h" + +namespace NSQLComplete { + + namespace { + + class TNameService: public INameService { + public: + explicit TNameService(ISchema::TPtr schema) + : Schema_(std::move(schema)) + { + } + + NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const override { + if (!request.Constraints.Object) { + return NThreading::MakeFuture<TNameResponse>({}); + } + + return Schema_ + ->List(ToListRequest(std::move(request))) + .Apply(ToNameResponse); + } + + private: + static TListRequest ToListRequest(TNameRequest request) { + return { + .Cluster = ClusterName(*request.Constraints.Object), + .Path = request.Prefix, + .Filter = ToListFilter(request.Constraints), + .Limit = request.Limit, + }; + } + + static TString ClusterName(const TObjectNameConstraints& constraints) { + TString name = constraints.Cluster; + if (!constraints.Provider.empty()) { + name.prepend(":"); + name.prepend(constraints.Provider); + } + return name; + } + + static TListFilter ToListFilter(const TNameConstraints& constraints) { + TListFilter filter; + filter.Types = THashSet<TString>(); + for (auto kind : constraints.Object->Kinds) { + filter.Types->emplace(ToFolderEntry(kind)); + } + return filter; + } + + static TString ToFolderEntry(EObjectKind kind) { + switch (kind) { + case EObjectKind::Folder: + return TFolderEntry::Folder; + case EObjectKind::Table: + return TFolderEntry::Table; + } + } + + static TNameResponse ToNameResponse(NThreading::TFuture<TListResponse> f) { + TListResponse list = f.ExtractValue(); + + TNameResponse response; + for (auto& entry : list.Entries) { + response.RankedNames.emplace_back(ToGenericName(std::move(entry))); + } + response.NameHintLength = list.NameHintLength; + return response; + } + + static TGenericName ToGenericName(TFolderEntry entry) { + TGenericName name; + if (entry.Type == TFolderEntry::Folder) { + TFolderName local; + local.Indentifier = std::move(entry.Name); + name = std::move(local); + } else if (entry.Type == TFolderEntry::Table) { + TTableName local; + local.Indentifier = std::move(entry.Name); + name = std::move(local); + } else { + TUnkownName local; + local.Content = std::move(entry.Name); + local.Type = std::move(entry.Type); + name = std::move(local); + } + return name; + } + + ISchema::TPtr Schema_; + }; + + } // namespace + + INameService::TPtr MakeSchemaNameService(ISchema::TPtr schema) { + return new TNameService(std::move(schema)); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/schema/name_service.h b/yql/essentials/sql/v1/complete/name/service/schema/name_service.h new file mode 100644 index 00000000000..aa2d7eb7f31 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/schema/name_service.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/object/schema.h> +#include <yql/essentials/sql/v1/complete/name/service/name_service.h> + +namespace NSQLComplete { + + INameService::TPtr MakeSchemaNameService(ISchema::TPtr schema); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/schema/ya.make b/yql/essentials/sql/v1/complete/name/service/schema/ya.make new file mode 100644 index 00000000000..9cdd3aad356 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/schema/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + name_service.cpp +) + +PEERDIR( + yql/essentials/sql/v1/complete/name/object + yql/essentials/sql/v1/complete/name/service +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp index c2373822f6f..0eadf446545 100644 --- a/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp @@ -21,6 +21,7 @@ namespace NSQLComplete { for (const auto& c : Children_) { fs.emplace_back(c->Lookup(request)); } + return NThreading::WaitAll(fs) .Apply([fs, this, request = std::move(request)](auto) { return Union(fs, request.Constraints, request.Limit); @@ -35,9 +36,17 @@ namespace NSQLComplete { TNameResponse united; for (auto f : fs) { TNameResponse response = f.ExtractValue(); + std::ranges::move( response.RankedNames, std::back_inserter(united.RankedNames)); + + if (!response.IsEmpty() && response.NameHintLength) { + Y_ENSURE( + united.NameHintLength.Empty() || + united.NameHintLength == response.NameHintLength); + united.NameHintLength = response.NameHintLength; + } } Ranking_->CropToSortedPrefix(united.RankedNames, constraints, limit); return united; diff --git a/yql/essentials/sql/v1/complete/name/service/ya.make b/yql/essentials/sql/v1/complete/name/service/ya.make index 1f1af9055ae..ec4de4d5e10 100644 --- a/yql/essentials/sql/v1/complete/name/service/ya.make +++ b/yql/essentials/sql/v1/complete/name/service/ya.make @@ -12,7 +12,9 @@ PEERDIR( END() RECURSE( + cluster ranking + schema static union ) diff --git a/yql/essentials/sql/v1/complete/name/ya.make b/yql/essentials/sql/v1/complete/name/ya.make index 8eb198ffa3d..0dcc75aabcc 100644 --- a/yql/essentials/sql/v1/complete/name/ya.make +++ b/yql/essentials/sql/v1/complete/name/ya.make @@ -8,6 +8,7 @@ PEERDIR( END() RECURSE( + cluster object service ) diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp index 0ec34e212db..d3941661e44 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete.cpp @@ -16,9 +16,9 @@ namespace NSQLComplete { TLexerSupplier lexer, INameService::TPtr names, ISqlCompletionEngine::TConfiguration configuration) - : Configuration(std::move(configuration)) - , SyntaxAnalysis(MakeLocalSyntaxAnalysis(lexer)) - , Names(std::move(names)) + : Configuration_(std::move(configuration)) + , SyntaxAnalysis_(MakeLocalSyntaxAnalysis(lexer)) + , Names_(std::move(names)) { } @@ -36,32 +36,35 @@ namespace NSQLComplete { << " for input size " << input.Text.size(); } - TLocalSyntaxContext context = SyntaxAnalysis->Analyze(input); + TLocalSyntaxContext context = SyntaxAnalysis_->Analyze(input); + auto keywords = context.Keywords; - TStringBuf prefix = input.Text.Head(input.CursorPosition); - TCompletedToken completedToken = GetCompletedToken(prefix); + TNameRequest request = NameRequestFrom(input, context); + if (request.IsEmpty()) { + return NThreading::MakeFuture<TCompletion>({ + .CompletedToken = GetCompletedToken(input, context.EditRange), + .Candidates = {}, + }); + } - return GetCandidates(std::move(context), completedToken) - .Apply([completedToken](NThreading::TFuture<TVector<TCandidate>> f) { - return TCompletion{ - .CompletedToken = std::move(completedToken), - .Candidates = f.ExtractValue(), - }; + return Names_->Lookup(std::move(request)) + .Apply([this, input, context = std::move(context)](auto f) { + return ToCompletion(input, context, f.ExtractValue()); }); } private: - TCompletedToken GetCompletedToken(TStringBuf prefix) const { + TCompletedToken GetCompletedToken(TCompletionInput input, TEditRange editRange) const { return { - .Content = LastWord(prefix), - .SourcePosition = LastWordIndex(prefix), + .Content = input.Text.SubStr(editRange.Begin, editRange.Length), + .SourcePosition = editRange.Begin, }; } - NThreading::TFuture<TVector<TCandidate>> GetCandidates(TLocalSyntaxContext context, const TCompletedToken& prefix) const { + TNameRequest NameRequestFrom(TCompletionInput input, const TLocalSyntaxContext& context) const { TNameRequest request = { - .Prefix = TString(prefix.Content), - .Limit = Configuration.Limit, + .Prefix = TString(GetCompletedToken(input, context.EditRange).Content), + .Limit = Configuration_.Limit, }; for (const auto& [first, _] : context.Keywords) { @@ -74,7 +77,7 @@ namespace NSQLComplete { request.Constraints.Pragma = std::move(constraints); } - if (context.IsTypeName) { + if (context.Type) { request.Constraints.Type = TTypeName::TConstraints(); } @@ -90,48 +93,109 @@ namespace NSQLComplete { request.Constraints.Hint = std::move(constraints); } - if (request.IsEmpty()) { - return NThreading::MakeFuture<TVector<TCandidate>>({}); + if (context.Object) { + request.Constraints.Object = TObjectNameConstraints{ + .Provider = context.Object->Provider, + .Cluster = context.Object->Cluster, + .Kinds = context.Object->Kinds, + }; + request.Prefix = context.Object->Path; } - return Names->Lookup(std::move(request)) - .Apply([keywords = std::move(context.Keywords)](NThreading::TFuture<TNameResponse> f) { - TNameResponse response = f.ExtractValue(); - return Convert(std::move(response.RankedNames), std::move(keywords)); - }); + if (context.Cluster) { + TClusterName::TConstraints constraints; + constraints.Namespace = context.Cluster->Provider; + request.Constraints.Cluster = std::move(constraints); + } + + return request; } - static TVector<TCandidate> Convert(TVector<TGenericName> names, TLocalSyntaxContext::TKeywords keywords) { + TCompletion ToCompletion( + TCompletionInput input, + TLocalSyntaxContext context, + TNameResponse response) const { + TCompletion completion = { + .CompletedToken = GetCompletedToken(input, context.EditRange), + .Candidates = Convert(std::move(response.RankedNames), std::move(context)), + }; + + if (response.NameHintLength) { + const auto length = *response.NameHintLength; + TEditRange editRange = { + .Begin = input.CursorPosition - length, + .Length = length, + }; + completion.CompletedToken = GetCompletedToken(input, editRange); + } + + return completion; + } + + static TVector<TCandidate> Convert(TVector<TGenericName> names, TLocalSyntaxContext context) { TVector<TCandidate> candidates; + candidates.reserve(names.size()); for (auto& name : names) { - candidates.emplace_back(std::visit([&](auto&& name) -> TCandidate { - using T = std::decay_t<decltype(name)>; - if constexpr (std::is_base_of_v<TKeyword, T>) { - TVector<TString>& seq = keywords[name.Content]; - seq.insert(std::begin(seq), name.Content); - return {ECandidateKind::Keyword, FormatKeywords(seq)}; - } - if constexpr (std::is_base_of_v<TPragmaName, T>) { - return {ECandidateKind::PragmaName, std::move(name.Indentifier)}; - } - if constexpr (std::is_base_of_v<TTypeName, T>) { - return {ECandidateKind::TypeName, std::move(name.Indentifier)}; - } - if constexpr (std::is_base_of_v<TFunctionName, T>) { - name.Indentifier += "("; - return {ECandidateKind::FunctionName, std::move(name.Indentifier)}; - } - if constexpr (std::is_base_of_v<THintName, T>) { - return {ECandidateKind::HintName, std::move(name.Indentifier)}; - } - }, std::move(name))); + candidates.emplace_back(Convert(std::move(name), context)); } return candidates; } - TConfiguration Configuration; - ILocalSyntaxAnalysis::TPtr SyntaxAnalysis; - INameService::TPtr Names; + static TCandidate Convert(TGenericName name, TLocalSyntaxContext& context) { + return std::visit([&](auto&& name) -> TCandidate { + using T = std::decay_t<decltype(name)>; + + if constexpr (std::is_base_of_v<TKeyword, T>) { + TVector<TString>& seq = context.Keywords[name.Content]; + seq.insert(std::begin(seq), name.Content); + return {ECandidateKind::Keyword, FormatKeywords(seq)}; + } + + if constexpr (std::is_base_of_v<TPragmaName, T>) { + return {ECandidateKind::PragmaName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<TTypeName, T>) { + return {ECandidateKind::TypeName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<TFunctionName, T>) { + name.Indentifier += "("; + return {ECandidateKind::FunctionName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<THintName, T>) { + return {ECandidateKind::HintName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<TFolderName, T>) { + name.Indentifier.append('/'); + if (!context.Object->IsEnclosed) { + name.Indentifier = Quoted(std::move(name.Indentifier)); + } + return {ECandidateKind::FolderName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<TTableName, T>) { + if (!context.Object->IsEnclosed) { + name.Indentifier = Quoted(std::move(name.Indentifier)); + } + return {ECandidateKind::TableName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<TClusterName, T>) { + return {ECandidateKind::ClusterName, std::move(name.Indentifier)}; + } + + if constexpr (std::is_base_of_v<TUnkownName, T>) { + return {ECandidateKind::UnknownName, std::move(name.Content)}; + } + }, std::move(name)); + } + + TConfiguration Configuration_; + ILocalSyntaxAnalysis::TPtr SyntaxAnalysis_; + INameService::TPtr Names_; }; ISqlCompletionEngine::TPtr MakeSqlCompletionEngine( @@ -162,6 +226,18 @@ void Out<NSQLComplete::ECandidateKind>(IOutputStream& out, NSQLComplete::ECandid case NSQLComplete::ECandidateKind::HintName: out << "HintName"; break; + case NSQLComplete::ECandidateKind::FolderName: + out << "FolderName"; + break; + case NSQLComplete::ECandidateKind::TableName: + out << "TableName"; + break; + case NSQLComplete::ECandidateKind::ClusterName: + out << "ClusterName"; + break; + case NSQLComplete::ECandidateKind::UnknownName: + out << "UnknownName"; + break; } } diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h index e74f3646ba9..1bc2c0ecf4e 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.h +++ b/yql/essentials/sql/v1/complete/sql_complete.h @@ -22,6 +22,10 @@ namespace NSQLComplete { TypeName, FunctionName, HintName, + FolderName, + TableName, + ClusterName, + UnknownName, }; struct TCandidate { diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp index 060dfd42add..a72446779cf 100644 --- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -1,8 +1,15 @@ #include "sql_complete.h" +#include <yql/essentials/sql/v1/complete/name/cluster/static/discovery.h> +#include <yql/essentials/sql/v1/complete/name/object/dispatch/schema.h> +#include <yql/essentials/sql/v1/complete/name/object/simple/schema.h> +#include <yql/essentials/sql/v1/complete/name/object/simple/static/schema.h> #include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h> #include <yql/essentials/sql/v1/complete/name/service/ranking/ranking.h> +#include <yql/essentials/sql/v1/complete/name/service/cluster/name_service.h> +#include <yql/essentials/sql/v1/complete/name/service/schema/name_service.h> #include <yql/essentials/sql/v1/complete/name/service/static/name_service.h> +#include <yql/essentials/sql/v1/complete/name/service/union/name_service.h> #include <yql/essentials/sql/v1/lexer/lexer.h> #include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h> @@ -14,26 +21,29 @@ using namespace NSQLComplete; -class TDummyException: public std::runtime_error { +class TDummyException: public yexception { public: - TDummyException() - : std::runtime_error("T_T") { + TDummyException() { + Append("T_T"); } }; class TFailingNameService: public INameService { public: - TFuture<TNameResponse> Lookup(TNameRequest) const override { + NThreading::TFuture<TNameResponse> Lookup(TNameRequest) const override { auto e = std::make_exception_ptr(TDummyException()); return NThreading::MakeErrorFuture<TNameResponse>(e); } }; Y_UNIT_TEST_SUITE(SqlCompleteTests) { + using ECandidateKind::ClusterName; + using ECandidateKind::FolderName; using ECandidateKind::FunctionName; using ECandidateKind::HintName; using ECandidateKind::Keyword; using ECandidateKind::PragmaName; + using ECandidateKind::TableName; using ECandidateKind::TypeName; TLexerSupplier MakePureLexerSupplier() { @@ -49,8 +59,13 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { ISqlCompletionEngine::TPtr MakeSqlCompletionEngineUT() { TLexerSupplier lexer = MakePureLexerSupplier(); + TNameSet names = { - .Pragmas = {"yson.CastToString"}, + .Pragmas = { + "yson.CastToString", + "yt.RuntimeCluster", + "yt.RuntimeClusterSelection", + }, .Types = {"Uint64"}, .Functions = { "StartsWith", @@ -62,27 +77,51 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { {EStatementKind::Insert, {"EXPIRATION"}}, }, }; - TFrequencyData frequency = {}; - INameService::TPtr service = MakeStaticNameService(std::move(names), std::move(frequency)); - return MakeSqlCompletionEngine(std::move(lexer), std::move(service)); - } - TCompletionInput SharpedInput(TString& text) { - constexpr char delim = '#'; + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fss = { + {"", {{"/", {{"Folder", "local"}, + {"Folder", "test"}, + {"Folder", "prod"}, + {"Folder", ".sys"}}}, + {"/local/", {{"Table", "example"}, + {"Table", "account"}, + {"Table", "abacaba"}}}, + {"/test/", {{"Folder", "service"}, + {"Table", "meta"}}}, + {"/test/service/", {{"Table", "example"}}}, + {"/.sys/", {{"Table", "status"}}}}}, + {"example", + {{"/", {{"Table", "people"}}}}}, + {"yt:saurus", + {{"/", {{"Table", "maxim"}}}}}, + }; - size_t pos = text.find_first_of(delim); - if (pos == TString::npos) { - return { - .Text = text, - }; + TVector<TString> clusters; + for (const auto& [cluster, _] : fss) { + clusters.emplace_back(cluster); + } + EraseIf(clusters, [](const auto& s) { return s.empty(); }); + + TFrequencyData frequency; + + IRanking::TPtr ranking = MakeDefaultRanking(frequency); + + THashMap<TString, ISchema::TPtr> schemasByCluster; + for (auto& [cluster, fs] : fss) { + schemasByCluster[std::move(cluster)] = + MakeSimpleSchema( + MakeStaticSimpleSchema(std::move(fs))); } - Y_ENSURE(!TStringBuf(text).Tail(pos + 1).Contains(delim)); - text.erase(std::begin(text) + pos); - return { - .Text = text, - .CursorPosition = pos, + TVector<INameService::TPtr> children = { + MakeStaticNameService(std::move(names), frequency), + MakeSchemaNameService(MakeDispatchSchema(std::move(schemasByCluster))), + MakeClusterNameService(MakeStaticClusterDiscovery(std::move(clusters))), }; + + INameService::TPtr service = MakeUnionNameService(std::move(children), ranking); + + return MakeSqlCompletionEngine(std::move(lexer), std::move(service)); } TVector<TCandidate> Complete(ISqlCompletionEngine::TPtr& engine, TString sharped) { @@ -141,6 +180,17 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { UNIT_ASSERT_VALUES_EQUAL(Complete(engine, ";"), expected); UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "; "), expected); UNIT_ASSERT_VALUES_EQUAL(Complete(engine, " ; "), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "#SELECT"), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "#SELECT * FROM"), expected); + } + + Y_UNIT_TEST(Use) { + TVector<TCandidate> expected = { + {ClusterName, "example"}, + {ClusterName, "yt:saurus"}, + }; + auto engine = MakeSqlCompletionEngineUT(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "USE "), expected); } Y_UNIT_TEST(Alter) { @@ -187,6 +237,28 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE "), expected); } + Y_UNIT_TEST(CreateTable) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {FolderName, "`.sys/`"}, + {FolderName, "`local/`"}, + {FolderName, "`prod/`"}, + {FolderName, "`test/`"}, + {ClusterName, "example"}, + {ClusterName, "yt:saurus"}, + {Keyword, "IF NOT EXISTS"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE TABLE #"), expected); + } + { + TVector<TCandidate> expected = { + {FolderName, "service/"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE TABLE `test/#`"), expected); + } + } + Y_UNIT_TEST(Delete) { TVector<TCandidate> expected = { {Keyword, "FROM"}, @@ -216,6 +288,21 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "DROP "), expected); } + Y_UNIT_TEST(DropObject) { + TVector<TCandidate> expected = { + {FolderName, "`.sys/`"}, + {FolderName, "`local/`"}, + {FolderName, "`prod/`"}, + {FolderName, "`test/`"}, + {ClusterName, "example"}, + {ClusterName, "yt:saurus"}, + {Keyword, "IF EXISTS"}, + }; + auto engine = MakeSqlCompletionEngineUT(); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "DROP TABLE "), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "DROP VIEW "), expected); + } + Y_UNIT_TEST(Explain) { TVector<TCandidate> expected = { {Keyword, "ALTER"}, @@ -299,7 +386,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { { TVector<TCandidate> expected = { {Keyword, "ANSI"}, - {PragmaName, "yson.CastToString"}}; + {PragmaName, "yson.CastToString"}, + {PragmaName, "yt.RuntimeCluster"}, + {PragmaName, "yt.RuntimeClusterSelection"}}; auto completion = engine->CompleteAsync({"PRAGMA "}).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, ""); @@ -332,6 +421,23 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected); UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "cast"); } + { + TVector<TCandidate> expected = { + {PragmaName, "RuntimeCluster"}, + {PragmaName, "RuntimeClusterSelection"}}; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "pragma yt."), expected); + UNIT_ASSERT_VALUES_EQUAL( + Complete(engine, "pragma yt.RuntimeClusterSelection='force';\npragma yt.Ru"), + expected); + } + { + TVector<TCandidate> expected = { + {PragmaName, "RuntimeCluster"}, + {PragmaName, "RuntimeClusterSelection"}}; + UNIT_ASSERT_VALUES_EQUAL( + Complete(engine, "pragma yt.Ru#\n"), + expected); + } } Y_UNIT_TEST(Select) { @@ -377,23 +483,132 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } Y_UNIT_TEST(SelectFrom) { - TVector<TCandidate> expected = { - {Keyword, "ANY"}, - {Keyword, "CALLABLE"}, - {Keyword, "DICT"}, - {Keyword, "ENUM"}, - {Keyword, "FLOW"}, - {Keyword, "LIST"}, - {Keyword, "OPTIONAL"}, - {Keyword, "RESOURCE"}, - {Keyword, "SET"}, - {Keyword, "STRUCT"}, - {Keyword, "TAGGED"}, - {Keyword, "TUPLE"}, - {Keyword, "VARIANT"}, - }; auto engine = MakeSqlCompletionEngineUT(); - UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM "), expected); + { + TVector<TCandidate> expected = { + {FolderName, "`.sys/`"}, + {FolderName, "`local/`"}, + {FolderName, "`prod/`"}, + {FolderName, "`test/`"}, + {ClusterName, "example"}, + {ClusterName, "yt:saurus"}, + {Keyword, "ANY"}, + {Keyword, "CALLABLE"}, + {Keyword, "DICT"}, + {Keyword, "ENUM"}, + {Keyword, "FLOW"}, + {Keyword, "LIST"}, + {Keyword, "OPTIONAL"}, + {Keyword, "RESOURCE"}, + {Keyword, "SET"}, + {Keyword, "STRUCT"}, + {Keyword, "TAGGED"}, + {Keyword, "TUPLE"}, + {Keyword, "VARIANT"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM "), expected); + } + { + TVector<TCandidate> expected = {}; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM `#"), expected); + } + { + TString input = "SELECT * FROM `#`"; + TVector<TCandidate> expected = { + {FolderName, ".sys/"}, + {FolderName, "local/"}, + {FolderName, "prod/"}, + {FolderName, "test/"}, + }; + TCompletion actual = engine->Complete(SharpedInput(input)); + UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, ""); + } + { + TString input = "SELECT * FROM `local/#`"; + TVector<TCandidate> expected = { + {TableName, "abacaba"}, + {TableName, "account"}, + {TableName, "example"}, + }; + TCompletion actual = engine->Complete(SharpedInput(input)); + UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, ""); + } + { + TString input = "SELECT * FROM `local/a#`"; + TVector<TCandidate> expected = { + {TableName, "abacaba"}, + {TableName, "account"}, + }; + TCompletion actual = engine->Complete(SharpedInput(input)); + UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, "a"); + } + { + TString input = "SELECT * FROM `.sy#`"; + TVector<TCandidate> expected = { + {FolderName, ".sys/"}, + }; + TCompletion actual = engine->Complete(SharpedInput(input)); + UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, ".sy"); + } + { + TString input = "SELECT * FROM `/test/ser#vice/`"; + TVector<TCandidate> expected = { + {FolderName, "service/"}, + }; + TCompletion actual = engine->Complete(SharpedInput(input)); + UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); + UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, "ser"); + } + } + + Y_UNIT_TEST(SelectFromCluster) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {ClusterName, "yt:saurus"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt#"), expected); + } + { + TVector<TCandidate> expected = { + {ClusterName, "saurus"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:"), expected); + } + { + TVector<TCandidate> expected = { + {ClusterName, "saurus"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:saurus#"), expected); + } + { + TVector<TCandidate> expected = { + {TableName, "`maxim`"}, + {Keyword, "CALLABLE"}, + {Keyword, "DICT"}, + {Keyword, "ENUM"}, + {Keyword, "FLOW"}, + {Keyword, "LIST"}, + {Keyword, "OPTIONAL"}, + {Keyword, "RESOURCE"}, + {Keyword, "SET"}, + {Keyword, "STRUCT"}, + {Keyword, "TAGGED"}, + {Keyword, "TUPLE"}, + {Keyword, "VARIANT"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:saurus."), expected); + } + { + TVector<TCandidate> expected = { + {TableName, "`people`"}, + }; + UNIT_ASSERT_VALUES_EQUAL(CompleteTop(1, engine, "SELECT * FROM example."), expected); + } } Y_UNIT_TEST(SelectWhere) { @@ -445,6 +660,28 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT "), expected); } + Y_UNIT_TEST(UpsertInto) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {FolderName, "`.sys/`"}, + {FolderName, "`local/`"}, + {FolderName, "`prod/`"}, + {FolderName, "`test/`"}, + {ClusterName, "example"}, + {ClusterName, "yt:saurus"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT INTO "), expected); + } + { + TVector<TCandidate> expected = { + {TableName, "meta"}, + {FolderName, "service/"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT INTO `test/#`"), expected); + } + } + Y_UNIT_TEST(TypeName) { TVector<TCandidate> expected = { {Keyword, "CALLABLE<("}, @@ -559,6 +796,52 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "INSERT INTO my_table WITH "), expected); } + Y_UNIT_TEST(CursorPosition) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {Keyword, "AND"}, + {Keyword, "AS"}, + {Keyword, "ASSUME"}, + {Keyword, "BETWEEN"}, + {Keyword, "COLLATE"}, + {Keyword, "EXCEPT"}, + {Keyword, "FROM"}, + {Keyword, "GLOB"}, + {Keyword, "GROUP"}, + {Keyword, "HAVING"}, + {Keyword, "ILIKE"}, + {Keyword, "IN"}, + {Keyword, "INTERSECT"}, + {Keyword, "INTO RESULT"}, + {Keyword, "IS"}, + {Keyword, "ISNULL"}, + {Keyword, "LIKE"}, + {Keyword, "LIMIT"}, + {Keyword, "MATCH"}, + {Keyword, "NOT"}, + {Keyword, "NOTNULL"}, + {Keyword, "OR"}, + {Keyword, "ORDER BY"}, + {Keyword, "REGEXP"}, + {Keyword, "RLIKE"}, + {Keyword, "UNION"}, + {Keyword, "WHERE"}, + {Keyword, "WINDOW"}, + {Keyword, "WITHOUT"}, + {Keyword, "XOR"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT `a`"), expected); + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT `a`#FROM"), expected); + } + { + TVector<TCandidate> expected = { + {Keyword, "FROM"}, + }; + UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM# "), expected); + } + } + Y_UNIT_TEST(Enclosed) { TVector<TCandidate> empty = {}; @@ -634,11 +917,11 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { wchar32 rune; while (ptr < end) { Y_ENSURE(ReadUTF8CharAndAdvance(rune, ptr, end) == RECODE_OK); - TCompletion completion = engine->CompleteAsync({ - .Text = query, - .CursorPosition = static_cast<size_t>(std::distance(begin, ptr)), - }) - .GetValueSync(); + TCompletionInput input = { + .Text = query, + .CursorPosition = static_cast<size_t>(std::distance(begin, ptr)), + }; + TCompletion completion = engine->CompleteAsync(input).GetValueSync(); Y_DO_NOT_OPTIMIZE_AWAY(completion); } } diff --git a/yql/essentials/sql/v1/complete/syntax/cursor_token_context.cpp b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.cpp new file mode 100644 index 00000000000..33aef36847a --- /dev/null +++ b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.cpp @@ -0,0 +1,160 @@ +#include "cursor_token_context.h" + +#include <yql/essentials/core/issue/yql_issue.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> + +namespace NSQLComplete { + + namespace { + + bool Tokenize(ILexer::TPtr& lexer, TCompletionInput input, TParsedTokenList& tokens) { + NYql::TIssues issues; + if (!NSQLTranslation::Tokenize( + *lexer, TString(input.Text), /* queryName = */ "", + tokens, issues, /* maxErrors = */ 1)) { + return false; + } + return true; + } + + TCursor GetCursor(const TParsedTokenList& tokens, size_t cursorPosition) { + size_t current = 0; + for (size_t i = 0; i < tokens.size() && current < cursorPosition; ++i) { + const auto& content = tokens[i].Content; + + current += content.size(); + if (current < cursorPosition) { + continue; + } + + TCursor cursor = { + .PrevTokenIndex = i, + .NextTokenIndex = i, + .Position = cursorPosition, + }; + + if (current == cursorPosition) { + cursor.NextTokenIndex += 1; + } + + return cursor; + } + + return { + .PrevTokenIndex = Nothing(), + .NextTokenIndex = 0, + .Position = cursorPosition, + }; + } + + TVector<size_t> GetTokenPositions(const TParsedTokenList& tokens) { + TVector<size_t> positions; + positions.reserve(tokens.size()); + size_t pos = 0; + for (const auto& token : tokens) { + positions.emplace_back(pos); + pos += token.Content.size(); + } + return positions; + } + + } // namespace + + bool TRichParsedToken::IsLiteral() const { + return Base->Name == "STRING_VALUE" || + Base->Name == "DIGIGTS" || + Base->Name == "INTEGER_VALUE" || + Base->Name == "REAL"; + } + + TRichParsedToken TokenAt(const TCursorTokenContext& context, size_t index) { + return { + .Base = &context.Tokens.at(index), + .Index = index, + .Position = context.TokenPositions.at(index), + }; + } + + TMaybe<TRichParsedToken> TCursorTokenContext::Enclosing() const { + if (Tokens.size() == 1) { + Y_ENSURE(Tokens[0].Name == "EOF"); + return Nothing(); + } + + if (Cursor.PrevTokenIndex.Empty()) { + return Nothing(); + } + + auto token = TokenAt(*this, *Cursor.PrevTokenIndex); + if (Cursor.PrevTokenIndex == Cursor.NextTokenIndex || + !IsWordBoundary(token.Base->Content.back())) { + return token; + } + + return Nothing(); + } + + TMaybe<TRichParsedToken> TCursorTokenContext::MatchCursorPrefix(const TVector<TStringBuf>& pattern) const { + const auto prefix = std::span{Tokens.begin(), Cursor.NextTokenIndex}; + if (prefix.size() < pattern.size()) { + return Nothing(); + } + + ssize_t i = static_cast<ssize_t>(prefix.size()) - 1; + ssize_t j = static_cast<ssize_t>(pattern.size()) - 1; + for (; 0 <= j; --i, --j) { + if (!pattern[j].empty() && prefix[i].Name != pattern[j]) { + return Nothing(); + } + } + return TokenAt(*this, prefix.size() - pattern.size()); + } + + bool GetStatement( + ILexer::TPtr& lexer, + TCompletionInput input, + TCompletionInput& output, + size_t& output_position) { + TVector<TString> statements; + NYql::TIssues issues; + if (!NSQLTranslationV1::SplitQueryToStatements( + TString(input.Text) + ";", lexer, + statements, issues, /* file = */ "", + /* areBlankSkipped = */ false)) { + return false; + } + + size_t& cursor = output_position; + cursor = 0; + for (const auto& statement : statements) { + if (input.CursorPosition < cursor + statement.size()) { + output = { + .Text = input.Text.SubStr(cursor, statement.size()), + .CursorPosition = input.CursorPosition - cursor, + }; + return true; + } + cursor += statement.size(); + } + + output = input; + return true; + } + + bool GetCursorTokenContext(ILexer::TPtr& lexer, TCompletionInput input, TCursorTokenContext& context) { + TParsedTokenList tokens; + if (!Tokenize(lexer, input, tokens)) { + return false; + } + + TVector<size_t> positions = GetTokenPositions(tokens); + TCursor cursor = GetCursor(tokens, input.CursorPosition); + context = { + .Tokens = std::move(tokens), + .TokenPositions = std::move(positions), + .Cursor = cursor, + }; + return true; + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/cursor_token_context.h b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.h new file mode 100644 index 00000000000..35d22231e35 --- /dev/null +++ b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.h @@ -0,0 +1,50 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/core/input.h> +#include <yql/essentials/sql/v1/complete/text/word.h> + +#include <yql/essentials/parser/lexer_common/lexer.h> + +#include <util/generic/maybe.h> + +namespace NSQLComplete { + + using NSQLTranslation::ILexer; + using NSQLTranslation::TParsedToken; + using NSQLTranslation::TParsedTokenList; + + struct TCursor { + TMaybe<size_t> PrevTokenIndex = Nothing(); + size_t NextTokenIndex = PrevTokenIndex ? *PrevTokenIndex : 0; + size_t Position = 0; + }; + + struct TRichParsedToken { + const TParsedToken* Base = nullptr; + size_t Index = 0; + size_t Position = 0; + + bool IsLiteral() const; + }; + + struct TCursorTokenContext { + TParsedTokenList Tokens; + TVector<size_t> TokenPositions; + TCursor Cursor; + + TMaybe<TRichParsedToken> Enclosing() const; + TMaybe<TRichParsedToken> MatchCursorPrefix(const TVector<TStringBuf>& pattern) const; + }; + + bool GetStatement( + ILexer::TPtr& lexer, + TCompletionInput input, + TCompletionInput& output, + size_t& output_position); + + bool GetCursorTokenContext( + ILexer::TPtr& lexer, + TCompletionInput input, + TCursorTokenContext& context); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/cursor_token_context_ut.cpp b/yql/essentials/sql/v1/complete/syntax/cursor_token_context_ut.cpp new file mode 100644 index 00000000000..0e275cca3b8 --- /dev/null +++ b/yql/essentials/sql/v1/complete/syntax/cursor_token_context_ut.cpp @@ -0,0 +1,50 @@ +#include "cursor_token_context.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> + +using namespace NSQLComplete; + +Y_UNIT_TEST_SUITE(CursorTokenContextTests) { + + NSQLTranslation::ILexer::TPtr MakeLexer() { + NSQLTranslationV1::TLexers lexers; + lexers.Antlr4Pure = NSQLTranslationV1::MakeAntlr4PureLexerFactory(); + return NSQLTranslationV1::MakeLexer( + lexers, /* ansi = */ false, /* antlr4 = */ true, + NSQLTranslationV1::ELexerFlavor::Pure); + } + + TCursorTokenContext Context(TString input) { + auto lexer = MakeLexer(); + TCursorTokenContext context; + UNIT_ASSERT(GetCursorTokenContext(lexer, SharpedInput(input), context)); + return context; + } + + Y_UNIT_TEST(Empty) { + auto context = Context(""); + UNIT_ASSERT(context.Cursor.PrevTokenIndex.Empty()); + UNIT_ASSERT_VALUES_EQUAL(context.Cursor.NextTokenIndex, 0); + UNIT_ASSERT_VALUES_EQUAL(context.Cursor.Position, 0); + UNIT_ASSERT(context.Enclosing().Empty()); + } + + Y_UNIT_TEST(Blank) { + UNIT_ASSERT(Context("# ").Enclosing().Empty()); + UNIT_ASSERT(Context(" #").Enclosing().Empty()); + UNIT_ASSERT(Context(" # ").Enclosing().Empty()); + } + + Y_UNIT_TEST(Enclosing) { + UNIT_ASSERT(Context("se#").Enclosing().Defined()); + UNIT_ASSERT(Context("#se").Enclosing().Empty()); + UNIT_ASSERT(Context("`se`#").Enclosing().Empty()); + UNIT_ASSERT(Context("#`se`").Enclosing().Empty()); + UNIT_ASSERT(Context("`se`#`se`").Enclosing().Defined()); + UNIT_ASSERT(Context("\"se\"#\"se\"").Enclosing().Empty()); + } + +} // Y_UNIT_TEST_SUITE(CursorTokenContextTests) diff --git a/yql/essentials/sql/v1/complete/syntax/format.cpp b/yql/essentials/sql/v1/complete/syntax/format.cpp index 1c9f146c923..43c36aea9dd 100644 --- a/yql/essentials/sql/v1/complete/syntax/format.cpp +++ b/yql/essentials/sql/v1/complete/syntax/format.cpp @@ -35,4 +35,17 @@ namespace NSQLComplete { return text; } + TString Quoted(TString content) { + content.prepend('`'); + content.append('`'); + return content; + } + + TString Unquoted(TString content) { + Y_ENSURE(2 <= content.size() && content.front() == '`' && content.back() == '`'); + content.erase(0, 1); + content.pop_back(); + return content; + } + } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/format.h b/yql/essentials/sql/v1/complete/syntax/format.h index 6c2f1b72ac2..58e5d1f1e4a 100644 --- a/yql/essentials/sql/v1/complete/syntax/format.h +++ b/yql/essentials/sql/v1/complete/syntax/format.h @@ -6,5 +6,7 @@ namespace NSQLComplete { TString FormatKeywords(const TVector<TString>& seq); + TString Quoted(TString content); + TString Unquoted(TString content); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/grammar.cpp b/yql/essentials/sql/v1/complete/syntax/grammar.cpp index 252deaf682c..c080fae5ae4 100644 --- a/yql/essentials/sql/v1/complete/syntax/grammar.cpp +++ b/yql/essentials/sql/v1/complete/syntax/grammar.cpp @@ -7,31 +7,31 @@ namespace NSQLComplete { class TSqlGrammar: public ISqlGrammar { public: TSqlGrammar(const NSQLReflect::TLexerGrammar& grammar) - : Parser(MakeDummyParser()) - , AllTokens(ComputeAllTokens()) - , KeywordTokens(ComputeKeywordTokens(grammar)) - , PunctuationTokens(ComputePunctuationTokens(grammar)) + : Parser_(MakeDummyParser()) + , AllTokens_(ComputeAllTokens()) + , KeywordTokens_(ComputeKeywordTokens(grammar)) + , PunctuationTokens_(ComputePunctuationTokens(grammar)) { } const antlr4::dfa::Vocabulary& GetVocabulary() const override { - return Parser->getVocabulary(); + return Parser_->getVocabulary(); } const std::unordered_set<TTokenId>& GetAllTokens() const override { - return AllTokens; + return AllTokens_; } const std::unordered_set<TTokenId>& GetKeywordTokens() const override { - return KeywordTokens; + return KeywordTokens_; } const std::unordered_set<TTokenId>& GetPunctuationTokens() const override { - return PunctuationTokens; + return PunctuationTokens_; } const std::string& SymbolizedRule(TRuleId rule) const override { - return Parser->getRuleNames().at(rule); + return Parser_->getRuleNames().at(rule); } private: @@ -76,10 +76,10 @@ namespace NSQLComplete { return punctuationTokens; } - const THolder<antlr4::Parser> Parser; - const std::unordered_set<TTokenId> AllTokens; - const std::unordered_set<TTokenId> KeywordTokens; - const std::unordered_set<TTokenId> PunctuationTokens; + const THolder<antlr4::Parser> Parser_; + const std::unordered_set<TTokenId> AllTokens_; + const std::unordered_set<TTokenId> KeywordTokens_; + const std::unordered_set<TTokenId> PunctuationTokens_; }; const ISqlGrammar& GetSqlGrammar() { diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp index c434fa28daf..549208d4cab 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.cpp +++ b/yql/essentials/sql/v1/complete/syntax/local.cpp @@ -1,9 +1,10 @@ #include "local.h" #include "ansi.h" +#include "cursor_token_context.h" +#include "format.h" #include "grammar.h" #include "parser_call_stack.h" -#include "token.h" #include <yql/essentials/sql/v1/complete/antlr4/c3i.h> #include <yql/essentials/sql/v1/complete/antlr4/c3t.h> @@ -49,65 +50,77 @@ namespace NSQLComplete { public: explicit TSpecializedLocalSyntaxAnalysis(TLexerSupplier lexer) - : Grammar(&GetSqlGrammar()) + : Grammar_(&GetSqlGrammar()) , Lexer_(lexer(/* ansi = */ IsAnsiLexer)) - , C3(ComputeC3Config()) + , C3_(ComputeC3Config()) { } TLocalSyntaxContext Analyze(TCompletionInput input) override { TCompletionInput statement; - if (!GetStatement(Lexer_, input, statement)) { + size_t statement_position; + if (!GetStatement(Lexer_, input, statement, statement_position)) { return {}; } - auto candidates = C3.Complete(statement); - - TParsedTokenList tokens; - TCaretTokenPosition caret; - if (!TokenizePrefix(statement, tokens, caret)) { + TCursorTokenContext context; + if (!GetCursorTokenContext(Lexer_, statement, context)) { return {}; } - if (IsCaretEnslosed(tokens, caret)) { - return {}; + TC3Candidates candidates = C3_.Complete(statement); + + TLocalSyntaxContext result; + + result.EditRange = EditRange(context); + result.EditRange.Begin += statement_position; + + if (auto enclosing = context.Enclosing()) { + if (enclosing->IsLiteral()) { + return result; + } else if (enclosing->Base->Name == "ID_QUOTED") { + result.Object = ObjectMatch(context, candidates); + return result; + } } - return { - .Keywords = SiftedKeywords(candidates), - .Pragma = PragmaMatch(tokens, candidates), - .IsTypeName = IsTypeNameMatched(candidates), - .Function = FunctionMatch(tokens, candidates), - .Hint = HintMatch(candidates), - }; + result.Keywords = SiftedKeywords(candidates); + result.Pragma = PragmaMatch(context, candidates); + result.Type = TypeMatch(candidates); + result.Function = FunctionMatch(context, candidates); + result.Hint = HintMatch(candidates); + result.Object = ObjectMatch(context, candidates); + result.Cluster = ClusterMatch(context, candidates); + + return result; } private: - IC3Engine::TConfig ComputeC3Config() { + IC3Engine::TConfig ComputeC3Config() const { return { .IgnoredTokens = ComputeIgnoredTokens(), .PreferredRules = ComputePreferredRules(), }; } - std::unordered_set<TTokenId> ComputeIgnoredTokens() { - auto ignoredTokens = Grammar->GetAllTokens(); - for (auto keywordToken : Grammar->GetKeywordTokens()) { + std::unordered_set<TTokenId> ComputeIgnoredTokens() const { + auto ignoredTokens = Grammar_->GetAllTokens(); + for (auto keywordToken : Grammar_->GetKeywordTokens()) { ignoredTokens.erase(keywordToken); } - for (auto punctuationToken : Grammar->GetPunctuationTokens()) { + for (auto punctuationToken : Grammar_->GetPunctuationTokens()) { ignoredTokens.erase(punctuationToken); } return ignoredTokens; } - std::unordered_set<TRuleId> ComputePreferredRules() { + std::unordered_set<TRuleId> ComputePreferredRules() const { return GetC3PreferredRules(); } - TLocalSyntaxContext::TKeywords SiftedKeywords(const TC3Candidates& candidates) { - const auto& vocabulary = Grammar->GetVocabulary(); - const auto& keywordTokens = Grammar->GetKeywordTokens(); + TLocalSyntaxContext::TKeywords SiftedKeywords(const TC3Candidates& candidates) const { + const auto& vocabulary = Grammar_->GetVocabulary(); + const auto& keywordTokens = Grammar_->GetKeywordTokens(); TLocalSyntaxContext::TKeywords keywords; for (const auto& token : candidates.Tokens) { @@ -122,40 +135,41 @@ namespace NSQLComplete { } TMaybe<TLocalSyntaxContext::TPragma> PragmaMatch( - const TParsedTokenList& tokens, const TC3Candidates& candidates) { + const TCursorTokenContext& context, const TC3Candidates& candidates) const { if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyPragmaStack))) { return Nothing(); } TLocalSyntaxContext::TPragma pragma; - if (EndsWith(tokens, {"ID_PLAIN", "DOT"})) { - pragma.Namespace = tokens[tokens.size() - 2].Content; - } else if (EndsWith(tokens, {"ID_PLAIN", "DOT", ""})) { - pragma.Namespace = tokens[tokens.size() - 3].Content; + + if (TMaybe<TRichParsedToken> begin; + (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT"})) || + (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT", ""}))) { + pragma.Namespace = begin->Base->Content; } return pragma; } - bool IsTypeNameMatched(const TC3Candidates& candidates) { + bool TypeMatch(const TC3Candidates& candidates) const { return AnyOf(candidates.Rules, RuleAdapted(IsLikelyTypeStack)); } TMaybe<TLocalSyntaxContext::TFunction> FunctionMatch( - const TParsedTokenList& tokens, const TC3Candidates& candidates) { + const TCursorTokenContext& context, const TC3Candidates& candidates) const { if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyFunctionStack))) { return Nothing(); } TLocalSyntaxContext::TFunction function; - if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE"})) { - function.Namespace = tokens[tokens.size() - 2].Content; - } else if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE", ""})) { - function.Namespace = tokens[tokens.size() - 3].Content; + if (TMaybe<TRichParsedToken> begin; + (begin = context.MatchCursorPrefix({"ID_PLAIN", "NAMESPACE"})) || + (begin = context.MatchCursorPrefix({"ID_PLAIN", "NAMESPACE", ""}))) { + function.Namespace = begin->Base->Content; } return function; } - TMaybe<TLocalSyntaxContext::THint> HintMatch(const TC3Candidates& candidates) { + TMaybe<TLocalSyntaxContext::THint> HintMatch(const TC3Candidates& candidates) const { // TODO(YQL-19747): detect local contexts with a single iteration through the candidates.Rules auto rule = FindIf(candidates.Rules, RuleAdapted(IsLikelyHintStack)); if (rule == std::end(candidates.Rules)) { @@ -172,45 +186,103 @@ namespace NSQLComplete { }; } - bool TokenizePrefix(TCompletionInput input, TParsedTokenList& tokens, TCaretTokenPosition& caret) { - NYql::TIssues issues; - if (!NSQLTranslation::Tokenize( - *Lexer_, TString(input.Text), /* queryName = */ "", - tokens, issues, /* maxErrors = */ 1)) { - return false; + TMaybe<TLocalSyntaxContext::TObject> ObjectMatch( + const TCursorTokenContext& context, const TC3Candidates& candidates) const { + TLocalSyntaxContext::TObject object; + + if (AnyOf(candidates.Rules, RuleAdapted(IsLikelyObjectRefStack))) { + object.Kinds.emplace(EObjectKind::Folder); + } + + if (AnyOf(candidates.Rules, RuleAdapted(IsLikelyExistingTableStack))) { + object.Kinds.emplace(EObjectKind::Folder); + object.Kinds.emplace(EObjectKind::Table); + } + + if (object.Kinds.empty()) { + return Nothing(); } - Y_ENSURE(!tokens.empty() && tokens.back().Name == "EOF"); - tokens.pop_back(); + if (TMaybe<TRichParsedToken> begin; + (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT"})) || + (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT", ""}))) { + object.Cluster = begin->Base->Content; + } - caret = CaretTokenPosition(tokens, input.CursorPosition); - tokens.crop(caret.NextTokenIndex + 1); - return true; + if (TMaybe<TRichParsedToken> begin; + (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON", "ID_PLAIN", "DOT"})) || + (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON", "ID_PLAIN", "DOT", ""}))) { + object.Provider = begin->Base->Content; + } + + if (auto path = ObjectPath(context)) { + object.Path = *path; + object.IsEnclosed = true; + } + + return object; + } + + TMaybe<TString> ObjectPath(const TCursorTokenContext& context) const { + if (auto enclosing = context.Enclosing()) { + TString path = enclosing->Base->Content; + if (enclosing->Base->Name == "ID_QUOTED") { + path = Unquoted(std::move(path)); + } + path.resize(context.Cursor.Position - enclosing->Position - 1); + return path; + } + return Nothing(); } - bool IsCaretEnslosed(const TParsedTokenList& tokens, TCaretTokenPosition caret) { - if (tokens.empty() || caret.PrevTokenIndex != caret.NextTokenIndex) { - return false; + TMaybe<TLocalSyntaxContext::TCluster> ClusterMatch( + const TCursorTokenContext& context, const TC3Candidates& candidates) const { + if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyClusterStack))) { + return Nothing(); } - const auto& token = tokens.back(); - return token.Name == "STRING_VALUE" || - token.Name == "ID_QUOTED" || - token.Name == "DIGIGTS" || - token.Name == "INTEGER_VALUE" || - token.Name == "REAL"; + TLocalSyntaxContext::TCluster cluster; + if (TMaybe<TRichParsedToken> begin; + (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON"})) || + (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON", ""}))) { + cluster.Provider = begin->Base->Content; + } + return cluster; + } + + TEditRange EditRange(const TCursorTokenContext& context) const { + if (auto enclosing = context.Enclosing()) { + return EditRange(*enclosing, context.Cursor); + } + + return { + .Begin = context.Cursor.Position, + .Length = 0, + }; + } + + TEditRange EditRange(const TRichParsedToken& token, const TCursor& cursor) const { + size_t begin = token.Position; + if (token.Base->Name == "NOT_EQUALS2") { + begin += 1; + } + + return { + .Begin = begin, + .Length = cursor.Position - begin, + }; } - const ISqlGrammar* Grammar; + const ISqlGrammar* Grammar_; NSQLTranslation::ILexer::TPtr Lexer_; - TC3Engine<G> C3; + TC3Engine<G> C3_; }; class TLocalSyntaxAnalysis: public ILocalSyntaxAnalysis { public: explicit TLocalSyntaxAnalysis(TLexerSupplier lexer) - : DefaultEngine(lexer) - , AnsiEngine(lexer) + : DefaultEngine_(lexer) + , AnsiEngine_(lexer) { } @@ -223,13 +295,13 @@ namespace NSQLComplete { private: ILocalSyntaxAnalysis& GetSpecializedEngine(bool isAnsiLexer) { if (isAnsiLexer) { - return AnsiEngine; + return AnsiEngine_; } - return DefaultEngine; + return DefaultEngine_; } - TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ false> DefaultEngine; - TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ true> AnsiEngine; + TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ false> DefaultEngine_; + TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ true> AnsiEngine_; }; ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis(TLexerSupplier lexer) { diff --git a/yql/essentials/sql/v1/complete/syntax/local.h b/yql/essentials/sql/v1/complete/syntax/local.h index d58b62c62cd..8f88d5aa71c 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.h +++ b/yql/essentials/sql/v1/complete/syntax/local.h @@ -1,15 +1,22 @@ #pragma once +#include <yql/essentials/sql/v1/complete/core/name.h> #include <yql/essentials/sql/v1/complete/sql_complete.h> #include <yql/essentials/sql/v1/lexer/lexer.h> #include <util/generic/string.h> #include <util/generic/hash.h> +#include <util/generic/hash_set.h> #include <util/generic/maybe.h> namespace NSQLComplete { + struct TEditRange { + size_t Begin = 0; + size_t Length = 0; + }; + struct TLocalSyntaxContext { using TKeywords = THashMap<TString, TVector<TString>>; @@ -25,11 +32,26 @@ namespace NSQLComplete { EStatementKind StatementKind; }; + struct TCluster { + TString Provider; + }; + + struct TObject { + TString Provider; + TString Cluster; + TString Path; + THashSet<EObjectKind> Kinds; + bool IsEnclosed = false; + }; + TKeywords Keywords; TMaybe<TPragma> Pragma; - bool IsTypeName = false; + bool Type = false; TMaybe<TFunction> Function; TMaybe<THint> Hint; + TMaybe<TObject> Object; + TMaybe<TCluster> Cluster; + TEditRange EditRange; }; class ILocalSyntaxAnalysis { diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp index 938483438b1..ce6c94306d4 100644 --- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp +++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp @@ -13,7 +13,7 @@ namespace NSQLComplete { - const TVector<TRuleId> KeywordRules = { + const TVector<TRuleId> PreferredRules = { RULE(Keyword), RULE(Keyword_expr_uncompat), RULE(Keyword_table_uncompat), @@ -24,27 +24,13 @@ namespace NSQLComplete { RULE(Keyword_hint_uncompat), RULE(Keyword_as_compat), RULE(Keyword_compat), - }; - - const TVector<TRuleId> PragmaNameRules = { - RULE(Opt_id_prefix_or_type), - RULE(An_id), - }; - - const TVector<TRuleId> TypeNameRules = { - RULE(Type_name_simple), RULE(An_id_or_type), - }; - - const TVector<TRuleId> FunctionNameRules = { + RULE(An_id), RULE(Id_expr), - RULE(An_id_or_type), RULE(Id_or_type), - }; - - const TVector<TRuleId> HintNameRules = { RULE(Id_hint), - RULE(An_id), + RULE(Opt_id_prefix_or_type), + RULE(Type_name_simple), }; TVector<std::string> Symbolized(const TParserCallStack& stack) { @@ -101,6 +87,26 @@ namespace NSQLComplete { Contains({RULE(External_call_param), RULE(An_id)}, stack); } + bool IsLikelyObjectRefStack(const TParserCallStack& stack) { + return Contains({RULE(Object_ref)}, stack); + } + + bool IsLikelyExistingTableStack(const TParserCallStack& stack) { + return !Contains({RULE(Create_table_stmt), + RULE(Simple_table_ref)}, stack) && + (Contains({RULE(Simple_table_ref), + RULE(Simple_table_ref_core), + RULE(Object_ref)}, stack) || + Contains({RULE(Single_source), + RULE(Table_ref), + RULE(Table_key), + RULE(Id_table_or_type)}, stack)); + } + + bool IsLikelyClusterStack(const TParserCallStack& stack) { + return Contains({RULE(Cluster_expr)}, stack); + } + TMaybe<EStatementKind> StatementKindOf(const TParserCallStack& stack) { for (TRuleId rule : std::ranges::views::reverse(stack)) { if (rule == RULE(Process_core) || rule == RULE(Reduce_core) || rule == RULE(Select_core)) { @@ -115,10 +121,7 @@ namespace NSQLComplete { std::unordered_set<TRuleId> GetC3PreferredRules() { std::unordered_set<TRuleId> preferredRules; - preferredRules.insert(std::begin(KeywordRules), std::end(KeywordRules)); - preferredRules.insert(std::begin(PragmaNameRules), std::end(PragmaNameRules)); - preferredRules.insert(std::begin(TypeNameRules), std::end(TypeNameRules)); - preferredRules.insert(std::begin(FunctionNameRules), std::end(FunctionNameRules)); + preferredRules.insert(std::begin(PreferredRules), std::end(PreferredRules)); return preferredRules; } diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h index d185b72d628..d44b824a05e 100644 --- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h +++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h @@ -15,6 +15,12 @@ namespace NSQLComplete { bool IsLikelyHintStack(const TParserCallStack& stack); + bool IsLikelyObjectRefStack(const TParserCallStack& stack); + + bool IsLikelyExistingTableStack(const TParserCallStack& stack); + + bool IsLikelyClusterStack(const TParserCallStack& stack); + TMaybe<EStatementKind> StatementKindOf(const TParserCallStack& stack); std::unordered_set<TRuleId> GetC3PreferredRules(); diff --git a/yql/essentials/sql/v1/complete/syntax/token.cpp b/yql/essentials/sql/v1/complete/syntax/token.cpp deleted file mode 100644 index b8aee3211c6..00000000000 --- a/yql/essentials/sql/v1/complete/syntax/token.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "token.h" - -#include <yql/essentials/core/issue/yql_issue.h> -#include <yql/essentials/sql/v1/lexer/lexer.h> - -namespace NSQLComplete { - - bool GetStatement(NSQLTranslation::ILexer::TPtr& lexer, TCompletionInput input, TCompletionInput& output) { - TVector<TString> statements; - NYql::TIssues issues; - if (!NSQLTranslationV1::SplitQueryToStatements( - TString(input.Text) + ";", lexer, - statements, issues, /* file = */ "", - /* areBlankSkipped = */ false)) { - return false; - } - - size_t cursor = 0; - for (const auto& statement : statements) { - if (input.CursorPosition < cursor + statement.size()) { - output = { - .Text = input.Text.SubStr(cursor, statement.size()), - .CursorPosition = input.CursorPosition - cursor, - }; - return true; - } - cursor += statement.size(); - } - - output = input; - return true; - } - - TCaretTokenPosition CaretTokenPosition(const TParsedTokenList& tokens, size_t cursorPosition) { - size_t cursor = 0; - for (size_t i = 0; i < tokens.size(); ++i) { - const auto& content = tokens[i].Content; - cursor += content.size(); - if (cursorPosition < cursor) { - return {i, i}; - } else if (cursorPosition == cursor && IsWordBoundary(content.back())) { - return {i, i + 1}; - } - } - return {std::max(tokens.size(), static_cast<size_t>(1)) - 1, tokens.size()}; - } - - bool EndsWith(const TParsedTokenList& tokens, const TVector<TStringBuf>& pattern) { - if (tokens.size() < pattern.size()) { - return false; - } - for (yssize_t i = tokens.ysize() - 1, j = pattern.ysize() - 1; 0 <= j; --i, --j) { - if (!pattern[j].empty() && tokens[i].Name != pattern[j]) { - return false; - } - } - return true; - } - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/token.h b/yql/essentials/sql/v1/complete/syntax/token.h deleted file mode 100644 index d1e215285a9..00000000000 --- a/yql/essentials/sql/v1/complete/syntax/token.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include <yql/essentials/sql/v1/complete/core/input.h> -#include <yql/essentials/sql/v1/complete/text/word.h> - -#include <yql/essentials/parser/lexer_common/lexer.h> - -namespace NSQLComplete { - - using NSQLTranslation::TParsedTokenList; - - // `PrevTokenIndex` = `NextTokenIndex`, iff caret is enclosed - struct TCaretTokenPosition { - size_t PrevTokenIndex; - size_t NextTokenIndex; - }; - - bool GetStatement(NSQLTranslation::ILexer::TPtr& lexer, TCompletionInput input, TCompletionInput& output); - - TCaretTokenPosition CaretTokenPosition(const TParsedTokenList& tokens, size_t cursorPosition); - - bool EndsWith(const TParsedTokenList& tokens, const TVector<TStringBuf>& pattern); - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/ut/ya.make b/yql/essentials/sql/v1/complete/syntax/ut/ya.make index e070185af9f..7e682c5bac0 100644 --- a/yql/essentials/sql/v1/complete/syntax/ut/ya.make +++ b/yql/essentials/sql/v1/complete/syntax/ut/ya.make @@ -2,6 +2,11 @@ UNITTEST_FOR(yql/essentials/sql/v1/complete/syntax) SRCS( grammar_ut.cpp + cursor_token_context_ut.cpp +) + +PEERDIR( + yql/essentials/sql/v1/lexer/antlr4_pure ) END() diff --git a/yql/essentials/sql/v1/complete/syntax/ya.make b/yql/essentials/sql/v1/complete/syntax/ya.make index 9e2e908454b..7f63e5b2374 100644 --- a/yql/essentials/sql/v1/complete/syntax/ya.make +++ b/yql/essentials/sql/v1/complete/syntax/ya.make @@ -2,11 +2,11 @@ LIBRARY() SRCS( ansi.cpp + cursor_token_context.cpp format.cpp grammar.cpp local.cpp parser_call_stack.cpp - token.cpp ) ADDINCL( @@ -21,6 +21,8 @@ PEERDIR( yql/essentials/sql/settings yql/essentials/sql/v1/lexer yql/essentials/sql/v1/reflect + yql/essentials/sql/v1/complete/core + yql/essentials/sql/v1/complete/text ) END() diff --git a/yql/essentials/sql/v1/complete/ut/ya.make b/yql/essentials/sql/v1/complete/ut/ya.make index fbb84f56f25..c978e6e6048 100644 --- a/yql/essentials/sql/v1/complete/ut/ya.make +++ b/yql/essentials/sql/v1/complete/ut/ya.make @@ -7,7 +7,14 @@ SRCS( PEERDIR( yql/essentials/sql/v1/lexer/antlr4_pure yql/essentials/sql/v1/lexer/antlr4_pure_ansi + yql/essentials/sql/v1/complete/name/cluster/static + yql/essentials/sql/v1/complete/name/object/dispatch + yql/essentials/sql/v1/complete/name/object/simple + yql/essentials/sql/v1/complete/name/object/simple/static + yql/essentials/sql/v1/complete/name/service/cluster + yql/essentials/sql/v1/complete/name/service/schema yql/essentials/sql/v1/complete/name/service/static + yql/essentials/sql/v1/complete/name/service/union ) END() diff --git a/yql/essentials/tools/yql_complete/yql_complete b/yql/essentials/tools/yql_complete/yql_complete new file mode 100644 index 00000000000..7fc1116ee5d --- /dev/null +++ b/yql/essentials/tools/yql_complete/yql_complete @@ -0,0 +1 @@ +/home/vityaman/.ya/build/symres/bbe5c007c4bcc83d4396e13689e6b39b/yql_complete
\ No newline at end of file |