diff options
| author | vityaman <[email protected]> | 2025-05-06 15:49:02 +0300 | 
|---|---|---|
| committer | robot-piglet <[email protected]> | 2025-05-06 16:04:08 +0300 | 
| commit | 9c3fdca51d8ae892c5ad8f6ef92df73fafc09e28 (patch) | |
| tree | 561c599fae4ea29b537a6958b65e1b052548edf2 /yql/essentials/sql | |
| parent | c131e959456f9f9a4adada5623ce3bae4097a8c1 (diff) | |
YQL-19747 Complete folder, table and cluster names
---
- Related to `YQL-19747`
- On top of https://github.com/ytsaurus/ytsaurus/pull/1253
- Related to https://github.com/ydb-platform/ydb/issues/9056
- Related to https://github.com/vityaman/ydb/issues/14
- Related to https://github.com/vityaman/ydb/issues/35
- Related to https://github.com/vityaman/ydb/issues/40
---
Pull Request resolved: https://github.com/ytsaurus/ytsaurus/pull/1257
commit_hash:0b842abb27184c88b8177beeea29fb1ea86b7a04
Diffstat (limited to 'yql/essentials/sql')
57 files changed, 1540 insertions, 395 deletions
diff --git a/yql/essentials/sql/v1/complete/antlr4/c3t.h b/yql/essentials/sql/v1/complete/antlr4/c3t.h index 35b1f714fa7..aca5ebf92e5 100644 --- a/yql/essentials/sql/v1/complete/antlr4/c3t.h +++ b/yql/essentials/sql/v1/complete/antlr4/c3t.h @@ -27,37 +27,37 @@ namespace NSQLComplete {      class TC3Engine: public IC3Engine {      public:          explicit TC3Engine(TConfig config) -            : Chars() -            , Lexer(&Chars) -            , Tokens(&Lexer) -            , Parser(&Tokens) -            , CompletionCore(&Parser) +            : Chars_() +            , Lexer_(&Chars_) +            , Tokens_(&Lexer_) +            , Parser_(&Tokens_) +            , CompletionCore_(&Parser_)          { -            Lexer.removeErrorListeners(); -            Parser.removeErrorListeners(); +            Lexer_.removeErrorListeners(); +            Parser_.removeErrorListeners(); -            CompletionCore.ignoredTokens = std::move(config.IgnoredTokens); -            CompletionCore.preferredRules = std::move(config.PreferredRules); +            CompletionCore_.ignoredTokens = std::move(config.IgnoredTokens); +            CompletionCore_.preferredRules = std::move(config.PreferredRules);          }          TC3Candidates Complete(TCompletionInput input) override {              auto prefix = input.Text.Head(input.CursorPosition);              Assign(prefix);              const auto caretTokenIndex = CaretTokenIndex(prefix); -            auto candidates = CompletionCore.collectCandidates(caretTokenIndex); +            auto candidates = CompletionCore_.collectCandidates(caretTokenIndex);              return Converted(std::move(candidates));          }      private:          void Assign(TStringBuf prefix) { -            Chars.load(prefix.Data(), prefix.Size(), /* lenient = */ false); -            Lexer.reset(); -            Tokens.setTokenSource(&Lexer); -            Tokens.fill(); +            Chars_.load(prefix.Data(), prefix.Size(), /* lenient = */ false); +            Lexer_.reset(); +            Tokens_.setTokenSource(&Lexer_); +            Tokens_.fill();          }          size_t CaretTokenIndex(TStringBuf prefix) { -            const auto tokensCount = Tokens.size(); +            const auto tokensCount = Tokens_.size();              if (2 <= tokensCount && !LastWord(prefix).Empty()) {                  return tokensCount - 2;              } @@ -76,11 +76,11 @@ namespace NSQLComplete {              return converted;          } -        antlr4::ANTLRInputStream Chars; -        G::TLexer Lexer; -        antlr4::BufferedTokenStream Tokens; -        G::TParser Parser; -        c3::CodeCompletionCore CompletionCore; +        antlr4::ANTLRInputStream Chars_; +        G::TLexer Lexer_; +        antlr4::BufferedTokenStream Tokens_; +        G::TParser Parser_; +        c3::CodeCompletionCore CompletionCore_;      };  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/input.cpp b/yql/essentials/sql/v1/complete/core/input.cpp new file mode 100644 index 00000000000..8eca3a28ee8 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/input.cpp @@ -0,0 +1,25 @@ +#include "input.h" + +#include <util/generic/yexception.h> + +namespace NSQLComplete { + +    TCompletionInput SharpedInput(TString& text) { +        constexpr char delim = '#'; + +        size_t pos = text.find_first_of(delim); +        if (pos == TString::npos) { +            return { +                .Text = text, +            }; +        } + +        Y_ENSURE(!TStringBuf(text).Tail(pos + 1).Contains(delim)); +        text.erase(std::begin(text) + pos); +        return { +            .Text = text, +            .CursorPosition = pos, +        }; +    } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/input.h b/yql/essentials/sql/v1/complete/core/input.h index 3bb609cbb22..d852736cb44 100644 --- a/yql/essentials/sql/v1/complete/core/input.h +++ b/yql/essentials/sql/v1/complete/core/input.h @@ -9,4 +9,6 @@ namespace NSQLComplete {          size_t CursorPosition = Text.length();      }; +    TCompletionInput SharpedInput(TString& text Y_LIFETIME_BOUND); +  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/name.h b/yql/essentials/sql/v1/complete/core/name.h new file mode 100644 index 00000000000..02524766de1 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/name.h @@ -0,0 +1,10 @@ +#pragma once + +namespace NSQLComplete { + +    enum class EObjectKind { +        Folder, +        Table, +    }; + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/ya.make b/yql/essentials/sql/v1/complete/core/ya.make index 9865d255c8f..8bc457f8f95 100644 --- a/yql/essentials/sql/v1/complete/core/ya.make +++ b/yql/essentials/sql/v1/complete/core/ya.make @@ -1,3 +1,7 @@  LIBRARY() +SRCS( +    input.cpp +) +  END() diff --git a/yql/essentials/sql/v1/complete/name/cluster/discovery.h b/yql/essentials/sql/v1/complete/name/cluster/discovery.h new file mode 100644 index 00000000000..6b496f15546 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/discovery.h @@ -0,0 +1,19 @@ +#pragma once + +#include <library/cpp/threading/future/core/future.h> + +#include <util/generic/ptr.h> + +namespace NSQLComplete { + +    using TClusterList = TVector<TString>; + +    class IClusterDiscovery: public TThrRefBase { +    public: +        using TPtr = TIntrusivePtr<IClusterDiscovery>; + +        virtual ~IClusterDiscovery() = default; +        virtual NThreading::TFuture<TClusterList> Query() const = 0; +    }; + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/cluster/static/discovery.cpp b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.cpp new file mode 100644 index 00000000000..7caee64c182 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.cpp @@ -0,0 +1,28 @@ +#include "discovery.h" + +namespace NSQLComplete { + +    namespace { + +        class TClusterDiscovery: public IClusterDiscovery { +        public: +            explicit TClusterDiscovery(TVector<TString> instances) +                : ClusterList_(std::move(instances)) +            { +            } + +            NThreading::TFuture<TClusterList> Query() const override { +                return NThreading::MakeFuture(ClusterList_); +            } + +        private: +            TVector<TString> ClusterList_; +        }; + +    } // namespace + +    IClusterDiscovery::TPtr MakeStaticClusterDiscovery(TVector<TString> instances) { +        return new TClusterDiscovery(std::move(instances)); +    } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/cluster/static/discovery.h b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.h new file mode 100644 index 00000000000..bfad0eed62f --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/static/discovery.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/cluster/discovery.h> + +namespace NSQLComplete { + +    IClusterDiscovery::TPtr MakeStaticClusterDiscovery(TVector<TString> instances); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/cluster/static/ya.make b/yql/essentials/sql/v1/complete/name/cluster/static/ya.make new file mode 100644 index 00000000000..567130a2ff0 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/static/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( +    discovery.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/complete/name/cluster +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/cluster/ya.make b/yql/essentials/sql/v1/complete/name/cluster/ya.make new file mode 100644 index 00000000000..5ea880aeadd --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/cluster/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +PEERDIR( +    library/cpp/threading/future +) + +END() + +RECURSE( +    static +) diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp new file mode 100644 index 00000000000..f6d79b280a0 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp @@ -0,0 +1,36 @@ +#include "schema.h" + +namespace NSQLComplete { + +    namespace { + +        class TSchema: public ISchema { +        public: +            explicit TSchema(THashMap<TString, ISchema::TPtr> mapping) +                : Mapping_(std::move(mapping)) +            { +            } + +            NThreading::TFuture<TListResponse> List(const TListRequest& request) const override { +                auto iter = Mapping_.find(request.Cluster); +                if (iter == std::end(Mapping_)) { +                    yexception e; +                    e << "unknown cluster '" << request.Cluster << "'"; +                    std::exception_ptr p = std::make_exception_ptr(e); +                    return NThreading::MakeErrorFuture<TListResponse>(p); +                } + +                return iter->second->List(request); +            } + +        private: +            THashMap<TString, ISchema::TPtr> Mapping_; +        }; + +    } // namespace + +    ISchema::TPtr MakeDispatchSchema(THashMap<TString, ISchema::TPtr> mapping) { +        return new TSchema(std::move(mapping)); +    } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h new file mode 100644 index 00000000000..517a3ad0af7 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/object/schema.h> + +namespace NSQLComplete { + +    ISchema::TPtr MakeDispatchSchema(THashMap<TString, ISchema::TPtr> mapping); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make b/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make new file mode 100644 index 00000000000..071bf5dff7d --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( +    schema.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/complete/name/object +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/object/schema_gateway.cpp b/yql/essentials/sql/v1/complete/name/object/schema.cpp index c802ddcb7a1..ba9e09b2ee7 100644 --- a/yql/essentials/sql/v1/complete/name/object/schema_gateway.cpp +++ b/yql/essentials/sql/v1/complete/name/object/schema.cpp @@ -1,4 +1,4 @@ -#include "schema_gateway.h" +#include "schema.h"  template <>  void Out<NSQLComplete::TFolderEntry>(IOutputStream& out, const NSQLComplete::TFolderEntry& entry) { diff --git a/yql/essentials/sql/v1/complete/name/object/schema_gateway.h b/yql/essentials/sql/v1/complete/name/object/schema.h index f9307bf495d..687f92d7e8a 100644 --- a/yql/essentials/sql/v1/complete/name/object/schema_gateway.h +++ b/yql/essentials/sql/v1/complete/name/object/schema.h @@ -41,11 +41,11 @@ namespace NSQLComplete {          TVector<TFolderEntry> Entries;      }; -    class ISchemaGateway: public TThrRefBase { +    class ISchema: public TThrRefBase {      public: -        using TPtr = TIntrusivePtr<ISchemaGateway>; +        using TPtr = TIntrusivePtr<ISchema>; -        virtual ~ISchemaGateway() = default; +        virtual ~ISchema() = default;          virtual NThreading::TFuture<TListResponse> List(const TListRequest& request) const = 0;      }; diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.cpp b/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp index e8e7bf3ccd9..c7b62946f64 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema.cpp @@ -1,4 +1,4 @@ -#include "schema_gateway.h" +#include "schema.h"  #include <util/charset/utf8.h> @@ -6,7 +6,7 @@ namespace NSQLComplete {      namespace { -        class TSimpleSchemaGateway: public ISchemaGateway { +        class TSimpleSchema: public ISchema {          private:              static auto FilterByName(TString name) {                  return [name = std::move(name)](auto f) { @@ -47,14 +47,20 @@ namespace NSQLComplete {              }          public: -            explicit TSimpleSchemaGateway(ISimpleSchemaGateway::TPtr simple) +            explicit TSimpleSchema(ISimpleSchema::TPtr simple)                  : Simple_(std::move(simple))              {              }              NThreading::TFuture<TListResponse> List(const TListRequest& request) const override {                  auto [path, name] = Simple_->Split(request.Path); -                return Simple_->List(TString(path)) + +                TString pathStr(path); +                if (!pathStr.StartsWith('/')) { +                    pathStr.prepend('/'); +                } + +                return Simple_->List(std::move(pathStr))                      .Apply(FilterByName(TString(name)))                      .Apply(FilterByTypes(std::move(request.Filter.Types)))                      .Apply(Crop(request.Limit)) @@ -62,13 +68,13 @@ namespace NSQLComplete {              }          private: -            ISimpleSchemaGateway::TPtr Simple_; +            ISimpleSchema::TPtr Simple_;          };      } // namespace -    ISchemaGateway::TPtr MakeSimpleSchemaGateway(ISimpleSchemaGateway::TPtr simple) { -        return ISchemaGateway::TPtr(new TSimpleSchemaGateway(std::move(simple))); +    ISchema::TPtr MakeSimpleSchema(ISimpleSchema::TPtr simple) { +        return ISchema::TPtr(new TSimpleSchema(std::move(simple)));      }  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.h b/yql/essentials/sql/v1/complete/name/object/simple/schema.h index 4b4671f1cca..67def573a73 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema.h @@ -1,6 +1,6 @@  #pragma once -#include <yql/essentials/sql/v1/complete/name/object/schema_gateway.h> +#include <yql/essentials/sql/v1/complete/name/object/schema.h>  namespace NSQLComplete { @@ -9,15 +9,15 @@ namespace NSQLComplete {          TStringBuf NameHint;      }; -    class ISimpleSchemaGateway: public TThrRefBase { +    class ISimpleSchema: public TThrRefBase {      public: -        using TPtr = TIntrusivePtr<ISimpleSchemaGateway>; +        using TPtr = TIntrusivePtr<ISimpleSchema>; -        virtual ~ISimpleSchemaGateway() = default; +        virtual ~ISimpleSchema() = default;          virtual TSplittedPath Split(TStringBuf path) const = 0;          virtual NThreading::TFuture<TVector<TFolderEntry>> List(TString folder) const = 0;      }; -    ISchemaGateway::TPtr MakeSimpleSchemaGateway(ISimpleSchemaGateway::TPtr simple); +    ISchema::TPtr MakeSimpleSchema(ISimpleSchema::TPtr simple);  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway_ut.cpp b/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp index 86c8118f197..954ecc4da75 100644 --- a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway_ut.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp @@ -1,12 +1,14 @@ -#include "schema_gateway.h" +#include "schema.h" + +#include <yql/essentials/sql/v1/complete/name/object/simple/static/schema.h>  #include <library/cpp/testing/unittest/registar.h>  using namespace NSQLComplete; -Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) { +Y_UNIT_TEST_SUITE(StaticSchemaTests) { -    ISchemaGateway::TPtr MakeStaticSchemaGatewayUT() { +    ISchema::TPtr MakeStaticSchemaUT() {          THashMap<TString, TVector<TFolderEntry>> fs = {              {"/", {{"Folder", "local"},                     {"Folder", "test"}, @@ -18,18 +20,19 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) {                          {"Table", "meta"}}},              {"/test/service/", {{"Table", "example"}}},          }; -        return MakeStaticSchemaGateway(std::move(fs)); +        return MakeSimpleSchema( +            MakeStaticSimpleSchema(std::move(fs)));      }      Y_UNIT_TEST(ListFolderBasic) { -        auto gateway = MakeStaticSchemaGatewayUT(); +        auto schema = MakeStaticSchemaUT();          {              TVector<TFolderEntry> expected = {                  {"Folder", "local"},                  {"Folder", "test"},                  {"Folder", "prod"},              }; -            UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/"}).GetValueSync().Entries, expected); +            UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/"}).GetValueSync().Entries, expected);          }          {              TVector<TFolderEntry> expected = { @@ -37,29 +40,29 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) {                  {"Table", "account"},                  {"Table", "abacaba"},              }; -            UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/local/"}).GetValueSync().Entries, expected); +            UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/local/"}).GetValueSync().Entries, expected);          }          {              TVector<TFolderEntry> expected = {                  {"Folder", "service"},                  {"Table", "meta"},              }; -            UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/test/"}).GetValueSync().Entries, expected); +            UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/test/"}).GetValueSync().Entries, expected);          }          {              TVector<TFolderEntry> expected = {                  {"Table", "example"}}; -            UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/test/service/"}).GetValueSync().Entries, expected); +            UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/test/service/"}).GetValueSync().Entries, expected);          }      }      Y_UNIT_TEST(ListFolderHint) { -        auto gateway = MakeStaticSchemaGatewayUT(); +        auto schema = MakeStaticSchemaUT();          {              TVector<TFolderEntry> expected = {                  {"Folder", "local"},              }; -            auto actual = gateway->List({.Path = "/l"}).GetValueSync(); +            auto actual = schema->List({.Path = "/l"}).GetValueSync();              UNIT_ASSERT_VALUES_EQUAL(actual.Entries, expected);              UNIT_ASSERT_VALUES_EQUAL(actual.NameHintLength, 1);          } @@ -68,7 +71,7 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) {                  {"Table", "account"},                  {"Table", "abacaba"},              }; -            auto actual = gateway->List({.Path = "/local/a"}).GetValueSync(); +            auto actual = schema->List({.Path = "/local/a"}).GetValueSync();              UNIT_ASSERT_VALUES_EQUAL(actual.Entries, expected);              UNIT_ASSERT_VALUES_EQUAL(actual.NameHintLength, 1);          } @@ -76,14 +79,14 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) {              TVector<TFolderEntry> expected = {                  {"Folder", "service"},              }; -            auto actual = gateway->List({.Path = "/test/service"}).GetValueSync(); +            auto actual = schema->List({.Path = "/test/service"}).GetValueSync();              UNIT_ASSERT_VALUES_EQUAL(actual.Entries, expected);              UNIT_ASSERT_VALUES_EQUAL(actual.NameHintLength, 7);          }      }      Y_UNIT_TEST(ListFolderFilterByType) { -        auto gateway = MakeStaticSchemaGatewayUT(); +        auto schema = MakeStaticSchemaUT();          {              TVector<TFolderEntry> expected = {                  {"Folder", "service"}, @@ -94,7 +97,7 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) {                      .Types = THashSet<TString>{"Folder"},                  },              }; -            UNIT_ASSERT_VALUES_EQUAL(gateway->List(request).GetValueSync().Entries, expected); +            UNIT_ASSERT_VALUES_EQUAL(schema->List(request).GetValueSync().Entries, expected);          }          {              TVector<TFolderEntry> expected = { @@ -106,18 +109,18 @@ Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) {                      .Types = THashSet<TString>{"Table"},                  },              }; -            UNIT_ASSERT_VALUES_EQUAL(gateway->List(request).GetValueSync().Entries, expected); +            UNIT_ASSERT_VALUES_EQUAL(schema->List(request).GetValueSync().Entries, expected);          }      }      Y_UNIT_TEST(ListFolderLimit) { -        auto gateway = MakeStaticSchemaGatewayUT(); -        UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 0}).GetValueSync().Entries.size(), 0); -        UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 1}).GetValueSync().Entries.size(), 1); -        UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 2}).GetValueSync().Entries.size(), 2); -        UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 3}).GetValueSync().Entries.size(), 3); -        UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 4}).GetValueSync().Entries.size(), 3); -        UNIT_ASSERT_VALUES_EQUAL(gateway->List({.Path = "/", .Limit = 5}).GetValueSync().Entries.size(), 3); +        auto schema = MakeStaticSchemaUT(); +        UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 0}).GetValueSync().Entries.size(), 0); +        UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 1}).GetValueSync().Entries.size(), 1); +        UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 2}).GetValueSync().Entries.size(), 2); +        UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 3}).GetValueSync().Entries.size(), 3); +        UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 4}).GetValueSync().Entries.size(), 3); +        UNIT_ASSERT_VALUES_EQUAL(schema->List({.Path = "/", .Limit = 5}).GetValueSync().Entries.size(), 3);      } -} // Y_UNIT_TEST_SUITE(StaticSchemaGatewayTests) +} // Y_UNIT_TEST_SUITE(StaticSchemaTests) diff --git a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.cpp b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp index f43af57c752..0af3ff0ef96 100644 --- a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp @@ -1,16 +1,12 @@ -#include "schema_gateway.h" - -#include <yql/essentials/sql/v1/complete/name/object/simple/schema_gateway.h> - -#include <util/charset/utf8.h> +#include "schema.h"  namespace NSQLComplete {      namespace { -        class TSimpleSchemaGateway: public ISimpleSchemaGateway { +        class TSimpleSchema: public ISimpleSchema {          public: -            explicit TSimpleSchemaGateway(THashMap<TString, TVector<TFolderEntry>> data) +            explicit TSimpleSchema(THashMap<TString, TVector<TFolderEntry>> data)                  : Data_(std::move(data))              {                  for (const auto& [k, _] : Data_) { @@ -44,10 +40,8 @@ namespace NSQLComplete {      } // namespace -    ISchemaGateway::TPtr MakeStaticSchemaGateway(THashMap<TString, TVector<TFolderEntry>> fs) { -        return MakeSimpleSchemaGateway( -            ISimpleSchemaGateway::TPtr( -                new TSimpleSchemaGateway(std::move(fs)))); +    ISimpleSchema::TPtr MakeStaticSimpleSchema(THashMap<TString, TVector<TFolderEntry>> fs) { +        return new TSimpleSchema(std::move(fs));      }  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h new file mode 100644 index 00000000000..f04c89f0b23 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/object/simple/schema.h> + +namespace NSQLComplete { + +    ISimpleSchema::TPtr MakeStaticSimpleSchema( +        THashMap<TString, TVector<TFolderEntry>> fs); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/ya.make b/yql/essentials/sql/v1/complete/name/object/simple/static/ya.make new file mode 100644 index 00000000000..8e7918ed011 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( +    schema.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/complete/name/object/simple +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/object/simple/ut/ya.make b/yql/essentials/sql/v1/complete/name/object/simple/ut/ya.make new file mode 100644 index 00000000000..048dc38d7d8 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/object/simple/ut/ya.make @@ -0,0 +1,11 @@ +UNITTEST_FOR(yql/essentials/sql/v1/complete/name/object/simple) + +SRCS( +    schema_ut.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/complete/name/object/simple/static +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/object/simple/ya.make b/yql/essentials/sql/v1/complete/name/object/simple/ya.make index d3668fdb1fc..56eafc1b848 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/ya.make +++ b/yql/essentials/sql/v1/complete/name/object/simple/ya.make @@ -1,7 +1,7 @@  LIBRARY()  SRCS( -    schema_gateway.cpp +    schema.cpp  )  PEERDIR( @@ -9,3 +9,11 @@ PEERDIR(  )  END() + +RECURSE( +    static +) + +RECURSE_FOR_TESTS( +    ut +) diff --git a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.h b/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.h deleted file mode 100644 index fd23a956d25..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/static/schema_gateway.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include <yql/essentials/sql/v1/complete/name/object/schema_gateway.h> - -namespace NSQLComplete { - -    ISchemaGateway::TPtr MakeStaticSchemaGateway(THashMap<TString, TVector<TFolderEntry>> fs); - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/static/ut/ya.make b/yql/essentials/sql/v1/complete/name/object/static/ut/ya.make deleted file mode 100644 index 877f015d806..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/static/ut/ya.make +++ /dev/null @@ -1,7 +0,0 @@ -UNITTEST_FOR(yql/essentials/sql/v1/complete/name/object/static) - -SRCS( -    schema_gateway_ut.cpp -) - -END() diff --git a/yql/essentials/sql/v1/complete/name/object/static/ya.make b/yql/essentials/sql/v1/complete/name/object/static/ya.make deleted file mode 100644 index d37495f0700..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/static/ya.make +++ /dev/null @@ -1,16 +0,0 @@ -LIBRARY() - -SRCS( -    schema_gateway.cpp -) - -PEERDIR( -    yql/essentials/sql/v1/complete/name/object -    yql/essentials/sql/v1/complete/name/object/simple -) - -END() - -RECURSE_FOR_TESTS( -    ut -) diff --git a/yql/essentials/sql/v1/complete/name/object/ya.make b/yql/essentials/sql/v1/complete/name/object/ya.make index 483f11c9a59..2561c018292 100644 --- a/yql/essentials/sql/v1/complete/name/object/ya.make +++ b/yql/essentials/sql/v1/complete/name/object/ya.make @@ -1,7 +1,7 @@  LIBRARY()  SRCS( -    schema_gateway.cpp +    schema.cpp  )  PEERDIR( @@ -11,6 +11,6 @@ PEERDIR(  END()  RECURSE( +    dispatch      simple -    static  ) diff --git a/yql/essentials/sql/v1/complete/name/service/cluster/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.cpp new file mode 100644 index 00000000000..db0ba00b667 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.cpp @@ -0,0 +1,82 @@ +#include "name_service.h" + +#include <util/charset/utf8.h> + +namespace NSQLComplete { + +    namespace { + +        class TNameService: public INameService { +        private: +            static auto FilterByName(TString name) { +                return [name = std::move(name)](auto f) { +                    TClusterList clusters = f.ExtractValue(); +                    EraseIf(clusters, [prefix = ToLowerUTF8(name)](const TString& instance) { +                        return !instance.StartsWith(prefix); +                    }); +                    return clusters; +                }; +            } + +            static auto Crop(size_t limit) { +                return [limit](auto f) { +                    TClusterList clusters = f.ExtractValue(); +                    clusters.crop(limit); +                    return clusters; +                }; +            } + +            static auto ToResponse(TNameConstraints constraints) { +                return [constraints = std::move(constraints)](auto f) { +                    TClusterList clusters = f.ExtractValue(); + +                    TNameResponse response; +                    response.RankedNames.reserve(clusters.size()); + +                    for (auto& cluster : clusters) { +                        TClusterName name; +                        name.Indentifier = std::move(cluster); +                        response.RankedNames.emplace_back(std::move(name)); +                    } + +                    response.RankedNames = constraints.Unqualified(std::move(response.RankedNames)); +                    return response; +                }; +            } + +        public: +            explicit TNameService(IClusterDiscovery::TPtr discovery) +                : Discovery_(std::move(discovery)) +            { +            } + +            NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const override { +                if (!request.Constraints.Cluster) { +                    return NThreading::MakeFuture<TNameResponse>({}); +                } + +                return Discovery_->Query() +                    .Apply(FilterByName(QualifiedClusterName(request))) +                    .Apply(Crop(request.Limit)) +                    .Apply(ToResponse(request.Constraints)); +            } + +        private: +            static TString QualifiedClusterName(const TNameRequest& request) { +                TClusterName cluster; +                cluster.Indentifier = request.Prefix; + +                TGenericName generic = request.Constraints.Qualified(cluster); +                return std::get<TClusterName>(std::move(generic)).Indentifier; +            } + +            IClusterDiscovery::TPtr Discovery_; +        }; + +    } // namespace + +    INameService::TPtr MakeClusterNameService(IClusterDiscovery::TPtr discovery) { +        return new TNameService(std::move(discovery)); +    } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/cluster/name_service.h b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.h new file mode 100644 index 00000000000..a57eabc0d2b --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/cluster/name_service.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/cluster/discovery.h> +#include <yql/essentials/sql/v1/complete/name/service/name_service.h> + +namespace NSQLComplete { + +    INameService::TPtr MakeClusterNameService(IClusterDiscovery::TPtr discovery); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/cluster/ya.make b/yql/essentials/sql/v1/complete/name/service/cluster/ya.make new file mode 100644 index 00000000000..4849690a6f2 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/cluster/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( +    name_service.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/complete/name/cluster +    yql/essentials/sql/v1/complete/name/service +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/service/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/name_service.cpp index 88473a60bff..92d8b50e98f 100644 --- a/yql/essentials/sql/v1/complete/name/service/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/name_service.cpp @@ -34,6 +34,8 @@ namespace NSQLComplete {                  SetPrefix(name.Indentifier, ".", *Pragma);              } else if constexpr (std::is_same_v<T, TFunctionName>) {                  SetPrefix(name.Indentifier, "::", *Function); +            } else if constexpr (std::is_same_v<T, TClusterName>) { +                SetPrefix(name.Indentifier, ":", *Cluster);              }              return name;          }, std::move(unqualified)); @@ -46,6 +48,8 @@ namespace NSQLComplete {                  FixPrefix(name.Indentifier, ".", *Pragma);              } else if constexpr (std::is_same_v<T, TFunctionName>) {                  FixPrefix(name.Indentifier, "::", *Function); +            } else if constexpr (std::is_same_v<T, TClusterName>) { +                FixPrefix(name.Indentifier, ":", *Cluster);              }              return name;          }, std::move(qualified)); diff --git a/yql/essentials/sql/v1/complete/name/service/name_service.h b/yql/essentials/sql/v1/complete/name/service/name_service.h index 7d773582b61..cdcfc3a4982 100644 --- a/yql/essentials/sql/v1/complete/name/service/name_service.h +++ b/yql/essentials/sql/v1/complete/name/service/name_service.h @@ -1,5 +1,6 @@  #pragma once +#include <yql/essentials/sql/v1/complete/core/name.h>  #include <yql/essentials/sql/v1/complete/core/statement.h>  #include <library/cpp/threading/future/core/future.h> @@ -7,11 +8,10 @@  #include <util/generic/vector.h>  #include <util/generic/string.h>  #include <util/generic/maybe.h> +#include <util/generic/hash_set.h>  namespace NSQLComplete { -    using NThreading::TFuture; // TODO(YQL-19747): remove -      struct TIndentifier {          TString Indentifier;      }; @@ -29,7 +29,7 @@ namespace NSQLComplete {      };      struct TTypeName: TIndentifier { -        using TConstraints = std::monostate; +        struct TConstraints {};      };      struct TFunctionName: TIndentifier { @@ -42,18 +42,45 @@ namespace NSQLComplete {          };      }; +    struct TObjectNameConstraints { +        TString Provider; +        TString Cluster; +        THashSet<EObjectKind> Kinds; +    }; + +    struct TFolderName: TIndentifier { +    }; + +    struct TTableName: TIndentifier { +    }; + +    struct TClusterName: TIndentifier { +        struct TConstraints: TNamespaced {}; +    }; + +    struct TUnkownName { +        TString Content; +        TString Type; +    }; +      using TGenericName = std::variant<          TKeyword,          TPragmaName,          TTypeName,          TFunctionName, -        THintName>; +        THintName, +        TFolderName, +        TTableName, +        TClusterName, +        TUnkownName>;      struct TNameConstraints {          TMaybe<TPragmaName::TConstraints> Pragma;          TMaybe<TTypeName::TConstraints> Type;          TMaybe<TFunctionName::TConstraints> Function;          TMaybe<THintName::TConstraints> Hint; +        TMaybe<TObjectNameConstraints> Object; +        TMaybe<TClusterName::TConstraints> Cluster;          TGenericName Qualified(TGenericName unqualified) const;          TGenericName Unqualified(TGenericName qualified) const; @@ -72,19 +99,26 @@ namespace NSQLComplete {                     !Constraints.Pragma &&                     !Constraints.Type &&                     !Constraints.Function && -                   !Constraints.Hint; +                   !Constraints.Hint && +                   !Constraints.Object && +                   !Constraints.Cluster;          }      };      struct TNameResponse {          TVector<TGenericName> RankedNames; +        TMaybe<size_t> NameHintLength; + +        bool IsEmpty() const { +            return RankedNames.empty(); +        }      };      class INameService: public TThrRefBase {      public:          using TPtr = TIntrusivePtr<INameService>; -        virtual TFuture<TNameResponse> Lookup(TNameRequest request) const = 0; +        virtual NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const = 0;          virtual ~INameService() = default;      }; diff --git a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp index 3e2dd322522..6b8aa42bc5e 100644 --- a/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp +++ b/yql/essentials/sql/v1/complete/name/service/ranking/ranking.cpp @@ -23,7 +23,7 @@ namespace NSQLComplete {              TVector<TGenericName>& names,              const TNameConstraints& constraints,              size_t limit) const override { -            limit = std::min(limit, names.size()); +            limit = Min(limit, names.size());              TVector<TRow> rows;              rows.reserve(names.size()); @@ -91,6 +91,15 @@ namespace NSQLComplete {                      }                  } +                if constexpr (std::is_same_v<T, TFolderName> || +                              std::is_same_v<T, TTableName>) { +                    return std::numeric_limits<size_t>::max(); +                } + +                if constexpr (std::is_same_v<T, TClusterName>) { +                    return std::numeric_limits<size_t>::max() - 8; +                } +                  return 0;              }, name);          } @@ -108,6 +117,9 @@ namespace NSQLComplete {                  if constexpr (std::is_base_of_v<TIndentifier, T>) {                      return name.Indentifier;                  } +                if constexpr (std::is_base_of_v<TUnkownName, T>) { +                    return name.Content; +                }              }, name);          } diff --git a/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp new file mode 100644 index 00000000000..de8e8db65ac --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp @@ -0,0 +1,100 @@ +#include "name_service.h" + +namespace NSQLComplete { + +    namespace { + +        class TNameService: public INameService { +        public: +            explicit TNameService(ISchema::TPtr schema) +                : Schema_(std::move(schema)) +            { +            } + +            NThreading::TFuture<TNameResponse> Lookup(TNameRequest request) const override { +                if (!request.Constraints.Object) { +                    return NThreading::MakeFuture<TNameResponse>({}); +                } + +                return Schema_ +                    ->List(ToListRequest(std::move(request))) +                    .Apply(ToNameResponse); +            } + +        private: +            static TListRequest ToListRequest(TNameRequest request) { +                return { +                    .Cluster = ClusterName(*request.Constraints.Object), +                    .Path = request.Prefix, +                    .Filter = ToListFilter(request.Constraints), +                    .Limit = request.Limit, +                }; +            } + +            static TString ClusterName(const TObjectNameConstraints& constraints) { +                TString name = constraints.Cluster; +                if (!constraints.Provider.empty()) { +                    name.prepend(":"); +                    name.prepend(constraints.Provider); +                } +                return name; +            } + +            static TListFilter ToListFilter(const TNameConstraints& constraints) { +                TListFilter filter; +                filter.Types = THashSet<TString>(); +                for (auto kind : constraints.Object->Kinds) { +                    filter.Types->emplace(ToFolderEntry(kind)); +                } +                return filter; +            } + +            static TString ToFolderEntry(EObjectKind kind) { +                switch (kind) { +                    case EObjectKind::Folder: +                        return TFolderEntry::Folder; +                    case EObjectKind::Table: +                        return TFolderEntry::Table; +                } +            } + +            static TNameResponse ToNameResponse(NThreading::TFuture<TListResponse> f) { +                TListResponse list = f.ExtractValue(); + +                TNameResponse response; +                for (auto& entry : list.Entries) { +                    response.RankedNames.emplace_back(ToGenericName(std::move(entry))); +                } +                response.NameHintLength = list.NameHintLength; +                return response; +            } + +            static TGenericName ToGenericName(TFolderEntry entry) { +                TGenericName name; +                if (entry.Type == TFolderEntry::Folder) { +                    TFolderName local; +                    local.Indentifier = std::move(entry.Name); +                    name = std::move(local); +                } else if (entry.Type == TFolderEntry::Table) { +                    TTableName local; +                    local.Indentifier = std::move(entry.Name); +                    name = std::move(local); +                } else { +                    TUnkownName local; +                    local.Content = std::move(entry.Name); +                    local.Type = std::move(entry.Type); +                    name = std::move(local); +                } +                return name; +            } + +            ISchema::TPtr Schema_; +        }; + +    } // namespace + +    INameService::TPtr MakeSchemaNameService(ISchema::TPtr schema) { +        return new TNameService(std::move(schema)); +    } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/schema/name_service.h b/yql/essentials/sql/v1/complete/name/service/schema/name_service.h new file mode 100644 index 00000000000..aa2d7eb7f31 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/schema/name_service.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/name/object/schema.h> +#include <yql/essentials/sql/v1/complete/name/service/name_service.h> + +namespace NSQLComplete { + +    INameService::TPtr MakeSchemaNameService(ISchema::TPtr schema); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/service/schema/ya.make b/yql/essentials/sql/v1/complete/name/service/schema/ya.make new file mode 100644 index 00000000000..9cdd3aad356 --- /dev/null +++ b/yql/essentials/sql/v1/complete/name/service/schema/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( +    name_service.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/complete/name/object +    yql/essentials/sql/v1/complete/name/service +) + +END() diff --git a/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp index c2373822f6f..0eadf446545 100644 --- a/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/union/name_service.cpp @@ -21,6 +21,7 @@ namespace NSQLComplete {                  for (const auto& c : Children_) {                      fs.emplace_back(c->Lookup(request));                  } +                  return NThreading::WaitAll(fs)                      .Apply([fs, this, request = std::move(request)](auto) {                          return Union(fs, request.Constraints, request.Limit); @@ -35,9 +36,17 @@ namespace NSQLComplete {                  TNameResponse united;                  for (auto f : fs) {                      TNameResponse response = f.ExtractValue(); +                      std::ranges::move(                          response.RankedNames,                          std::back_inserter(united.RankedNames)); + +                    if (!response.IsEmpty() && response.NameHintLength) { +                        Y_ENSURE( +                            united.NameHintLength.Empty() || +                            united.NameHintLength == response.NameHintLength); +                        united.NameHintLength = response.NameHintLength; +                    }                  }                  Ranking_->CropToSortedPrefix(united.RankedNames, constraints, limit);                  return united; diff --git a/yql/essentials/sql/v1/complete/name/service/ya.make b/yql/essentials/sql/v1/complete/name/service/ya.make index 1f1af9055ae..ec4de4d5e10 100644 --- a/yql/essentials/sql/v1/complete/name/service/ya.make +++ b/yql/essentials/sql/v1/complete/name/service/ya.make @@ -12,7 +12,9 @@ PEERDIR(  END()  RECURSE( +    cluster      ranking +    schema      static      union  ) diff --git a/yql/essentials/sql/v1/complete/name/ya.make b/yql/essentials/sql/v1/complete/name/ya.make index 8eb198ffa3d..0dcc75aabcc 100644 --- a/yql/essentials/sql/v1/complete/name/ya.make +++ b/yql/essentials/sql/v1/complete/name/ya.make @@ -8,6 +8,7 @@ PEERDIR(  END()  RECURSE( +    cluster      object      service  ) diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp index 0ec34e212db..d3941661e44 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete.cpp @@ -16,9 +16,9 @@ namespace NSQLComplete {              TLexerSupplier lexer,              INameService::TPtr names,              ISqlCompletionEngine::TConfiguration configuration) -            : Configuration(std::move(configuration)) -            , SyntaxAnalysis(MakeLocalSyntaxAnalysis(lexer)) -            , Names(std::move(names)) +            : Configuration_(std::move(configuration)) +            , SyntaxAnalysis_(MakeLocalSyntaxAnalysis(lexer)) +            , Names_(std::move(names))          {          } @@ -36,32 +36,35 @@ namespace NSQLComplete {                      << " for input size " << input.Text.size();              } -            TLocalSyntaxContext context = SyntaxAnalysis->Analyze(input); +            TLocalSyntaxContext context = SyntaxAnalysis_->Analyze(input); +            auto keywords = context.Keywords; -            TStringBuf prefix = input.Text.Head(input.CursorPosition); -            TCompletedToken completedToken = GetCompletedToken(prefix); +            TNameRequest request = NameRequestFrom(input, context); +            if (request.IsEmpty()) { +                return NThreading::MakeFuture<TCompletion>({ +                    .CompletedToken = GetCompletedToken(input, context.EditRange), +                    .Candidates = {}, +                }); +            } -            return GetCandidates(std::move(context), completedToken) -                .Apply([completedToken](NThreading::TFuture<TVector<TCandidate>> f) { -                    return TCompletion{ -                        .CompletedToken = std::move(completedToken), -                        .Candidates = f.ExtractValue(), -                    }; +            return Names_->Lookup(std::move(request)) +                .Apply([this, input, context = std::move(context)](auto f) { +                    return ToCompletion(input, context, f.ExtractValue());                  });          }      private: -        TCompletedToken GetCompletedToken(TStringBuf prefix) const { +        TCompletedToken GetCompletedToken(TCompletionInput input, TEditRange editRange) const {              return { -                .Content = LastWord(prefix), -                .SourcePosition = LastWordIndex(prefix), +                .Content = input.Text.SubStr(editRange.Begin, editRange.Length), +                .SourcePosition = editRange.Begin,              };          } -        NThreading::TFuture<TVector<TCandidate>> GetCandidates(TLocalSyntaxContext context, const TCompletedToken& prefix) const { +        TNameRequest NameRequestFrom(TCompletionInput input, const TLocalSyntaxContext& context) const {              TNameRequest request = { -                .Prefix = TString(prefix.Content), -                .Limit = Configuration.Limit, +                .Prefix = TString(GetCompletedToken(input, context.EditRange).Content), +                .Limit = Configuration_.Limit,              };              for (const auto& [first, _] : context.Keywords) { @@ -74,7 +77,7 @@ namespace NSQLComplete {                  request.Constraints.Pragma = std::move(constraints);              } -            if (context.IsTypeName) { +            if (context.Type) {                  request.Constraints.Type = TTypeName::TConstraints();              } @@ -90,48 +93,109 @@ namespace NSQLComplete {                  request.Constraints.Hint = std::move(constraints);              } -            if (request.IsEmpty()) { -                return NThreading::MakeFuture<TVector<TCandidate>>({}); +            if (context.Object) { +                request.Constraints.Object = TObjectNameConstraints{ +                    .Provider = context.Object->Provider, +                    .Cluster = context.Object->Cluster, +                    .Kinds = context.Object->Kinds, +                }; +                request.Prefix = context.Object->Path;              } -            return Names->Lookup(std::move(request)) -                .Apply([keywords = std::move(context.Keywords)](NThreading::TFuture<TNameResponse> f) { -                    TNameResponse response = f.ExtractValue(); -                    return Convert(std::move(response.RankedNames), std::move(keywords)); -                }); +            if (context.Cluster) { +                TClusterName::TConstraints constraints; +                constraints.Namespace = context.Cluster->Provider; +                request.Constraints.Cluster = std::move(constraints); +            } + +            return request;          } -        static TVector<TCandidate> Convert(TVector<TGenericName> names, TLocalSyntaxContext::TKeywords keywords) { +        TCompletion ToCompletion( +            TCompletionInput input, +            TLocalSyntaxContext context, +            TNameResponse response) const { +            TCompletion completion = { +                .CompletedToken = GetCompletedToken(input, context.EditRange), +                .Candidates = Convert(std::move(response.RankedNames), std::move(context)), +            }; + +            if (response.NameHintLength) { +                const auto length = *response.NameHintLength; +                TEditRange editRange = { +                    .Begin = input.CursorPosition - length, +                    .Length = length, +                }; +                completion.CompletedToken = GetCompletedToken(input, editRange); +            } + +            return completion; +        } + +        static TVector<TCandidate> Convert(TVector<TGenericName> names, TLocalSyntaxContext context) {              TVector<TCandidate> candidates; +            candidates.reserve(names.size());              for (auto& name : names) { -                candidates.emplace_back(std::visit([&](auto&& name) -> TCandidate { -                    using T = std::decay_t<decltype(name)>; -                    if constexpr (std::is_base_of_v<TKeyword, T>) { -                        TVector<TString>& seq = keywords[name.Content]; -                        seq.insert(std::begin(seq), name.Content); -                        return {ECandidateKind::Keyword, FormatKeywords(seq)}; -                    } -                    if constexpr (std::is_base_of_v<TPragmaName, T>) { -                        return {ECandidateKind::PragmaName, std::move(name.Indentifier)}; -                    } -                    if constexpr (std::is_base_of_v<TTypeName, T>) { -                        return {ECandidateKind::TypeName, std::move(name.Indentifier)}; -                    } -                    if constexpr (std::is_base_of_v<TFunctionName, T>) { -                        name.Indentifier += "("; -                        return {ECandidateKind::FunctionName, std::move(name.Indentifier)}; -                    } -                    if constexpr (std::is_base_of_v<THintName, T>) { -                        return {ECandidateKind::HintName, std::move(name.Indentifier)}; -                    } -                }, std::move(name))); +                candidates.emplace_back(Convert(std::move(name), context));              }              return candidates;          } -        TConfiguration Configuration; -        ILocalSyntaxAnalysis::TPtr SyntaxAnalysis; -        INameService::TPtr Names; +        static TCandidate Convert(TGenericName name, TLocalSyntaxContext& context) { +            return std::visit([&](auto&& name) -> TCandidate { +                using T = std::decay_t<decltype(name)>; + +                if constexpr (std::is_base_of_v<TKeyword, T>) { +                    TVector<TString>& seq = context.Keywords[name.Content]; +                    seq.insert(std::begin(seq), name.Content); +                    return {ECandidateKind::Keyword, FormatKeywords(seq)}; +                } + +                if constexpr (std::is_base_of_v<TPragmaName, T>) { +                    return {ECandidateKind::PragmaName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<TTypeName, T>) { +                    return {ECandidateKind::TypeName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<TFunctionName, T>) { +                    name.Indentifier += "("; +                    return {ECandidateKind::FunctionName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<THintName, T>) { +                    return {ECandidateKind::HintName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<TFolderName, T>) { +                    name.Indentifier.append('/'); +                    if (!context.Object->IsEnclosed) { +                        name.Indentifier = Quoted(std::move(name.Indentifier)); +                    } +                    return {ECandidateKind::FolderName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<TTableName, T>) { +                    if (!context.Object->IsEnclosed) { +                        name.Indentifier = Quoted(std::move(name.Indentifier)); +                    } +                    return {ECandidateKind::TableName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<TClusterName, T>) { +                    return {ECandidateKind::ClusterName, std::move(name.Indentifier)}; +                } + +                if constexpr (std::is_base_of_v<TUnkownName, T>) { +                    return {ECandidateKind::UnknownName, std::move(name.Content)}; +                } +            }, std::move(name)); +        } + +        TConfiguration Configuration_; +        ILocalSyntaxAnalysis::TPtr SyntaxAnalysis_; +        INameService::TPtr Names_;      };      ISqlCompletionEngine::TPtr MakeSqlCompletionEngine( @@ -162,6 +226,18 @@ void Out<NSQLComplete::ECandidateKind>(IOutputStream& out, NSQLComplete::ECandid          case NSQLComplete::ECandidateKind::HintName:              out << "HintName";              break; +        case NSQLComplete::ECandidateKind::FolderName: +            out << "FolderName"; +            break; +        case NSQLComplete::ECandidateKind::TableName: +            out << "TableName"; +            break; +        case NSQLComplete::ECandidateKind::ClusterName: +            out << "ClusterName"; +            break; +        case NSQLComplete::ECandidateKind::UnknownName: +            out << "UnknownName"; +            break;      }  } diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h index e74f3646ba9..1bc2c0ecf4e 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.h +++ b/yql/essentials/sql/v1/complete/sql_complete.h @@ -22,6 +22,10 @@ namespace NSQLComplete {          TypeName,          FunctionName,          HintName, +        FolderName, +        TableName, +        ClusterName, +        UnknownName,      };      struct TCandidate { diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp index 060dfd42add..a72446779cf 100644 --- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -1,8 +1,15 @@  #include "sql_complete.h" +#include <yql/essentials/sql/v1/complete/name/cluster/static/discovery.h> +#include <yql/essentials/sql/v1/complete/name/object/dispatch/schema.h> +#include <yql/essentials/sql/v1/complete/name/object/simple/schema.h> +#include <yql/essentials/sql/v1/complete/name/object/simple/static/schema.h>  #include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h>  #include <yql/essentials/sql/v1/complete/name/service/ranking/ranking.h> +#include <yql/essentials/sql/v1/complete/name/service/cluster/name_service.h> +#include <yql/essentials/sql/v1/complete/name/service/schema/name_service.h>  #include <yql/essentials/sql/v1/complete/name/service/static/name_service.h> +#include <yql/essentials/sql/v1/complete/name/service/union/name_service.h>  #include <yql/essentials/sql/v1/lexer/lexer.h>  #include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h> @@ -14,26 +21,29 @@  using namespace NSQLComplete; -class TDummyException: public std::runtime_error { +class TDummyException: public yexception {  public: -    TDummyException() -        : std::runtime_error("T_T") { +    TDummyException() { +        Append("T_T");      }  };  class TFailingNameService: public INameService {  public: -    TFuture<TNameResponse> Lookup(TNameRequest) const override { +    NThreading::TFuture<TNameResponse> Lookup(TNameRequest) const override {          auto e = std::make_exception_ptr(TDummyException());          return NThreading::MakeErrorFuture<TNameResponse>(e);      }  };  Y_UNIT_TEST_SUITE(SqlCompleteTests) { +    using ECandidateKind::ClusterName; +    using ECandidateKind::FolderName;      using ECandidateKind::FunctionName;      using ECandidateKind::HintName;      using ECandidateKind::Keyword;      using ECandidateKind::PragmaName; +    using ECandidateKind::TableName;      using ECandidateKind::TypeName;      TLexerSupplier MakePureLexerSupplier() { @@ -49,8 +59,13 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {      ISqlCompletionEngine::TPtr MakeSqlCompletionEngineUT() {          TLexerSupplier lexer = MakePureLexerSupplier(); +          TNameSet names = { -            .Pragmas = {"yson.CastToString"}, +            .Pragmas = { +                "yson.CastToString", +                "yt.RuntimeCluster", +                "yt.RuntimeClusterSelection", +            },              .Types = {"Uint64"},              .Functions = {                  "StartsWith", @@ -62,27 +77,51 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {                  {EStatementKind::Insert, {"EXPIRATION"}},              },          }; -        TFrequencyData frequency = {}; -        INameService::TPtr service = MakeStaticNameService(std::move(names), std::move(frequency)); -        return MakeSqlCompletionEngine(std::move(lexer), std::move(service)); -    } -    TCompletionInput SharpedInput(TString& text) { -        constexpr char delim = '#'; +        THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fss = { +            {"", {{"/", {{"Folder", "local"}, +                         {"Folder", "test"}, +                         {"Folder", "prod"}, +                         {"Folder", ".sys"}}}, +                  {"/local/", {{"Table", "example"}, +                               {"Table", "account"}, +                               {"Table", "abacaba"}}}, +                  {"/test/", {{"Folder", "service"}, +                              {"Table", "meta"}}}, +                  {"/test/service/", {{"Table", "example"}}}, +                  {"/.sys/", {{"Table", "status"}}}}}, +            {"example", +             {{"/", {{"Table", "people"}}}}}, +            {"yt:saurus", +             {{"/", {{"Table", "maxim"}}}}}, +        }; -        size_t pos = text.find_first_of(delim); -        if (pos == TString::npos) { -            return { -                .Text = text, -            }; +        TVector<TString> clusters; +        for (const auto& [cluster, _] : fss) { +            clusters.emplace_back(cluster); +        } +        EraseIf(clusters, [](const auto& s) { return s.empty(); }); + +        TFrequencyData frequency; + +        IRanking::TPtr ranking = MakeDefaultRanking(frequency); + +        THashMap<TString, ISchema::TPtr> schemasByCluster; +        for (auto& [cluster, fs] : fss) { +            schemasByCluster[std::move(cluster)] = +                MakeSimpleSchema( +                    MakeStaticSimpleSchema(std::move(fs)));          } -        Y_ENSURE(!TStringBuf(text).Tail(pos + 1).Contains(delim)); -        text.erase(std::begin(text) + pos); -        return { -            .Text = text, -            .CursorPosition = pos, +        TVector<INameService::TPtr> children = { +            MakeStaticNameService(std::move(names), frequency), +            MakeSchemaNameService(MakeDispatchSchema(std::move(schemasByCluster))), +            MakeClusterNameService(MakeStaticClusterDiscovery(std::move(clusters))),          }; + +        INameService::TPtr service = MakeUnionNameService(std::move(children), ranking); + +        return MakeSqlCompletionEngine(std::move(lexer), std::move(service));      }      TVector<TCandidate> Complete(ISqlCompletionEngine::TPtr& engine, TString sharped) { @@ -141,6 +180,17 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, ";"), expected);          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "; "), expected);          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, " ; "), expected); +        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "#SELECT"), expected); +        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "#SELECT * FROM"), expected); +    } + +    Y_UNIT_TEST(Use) { +        TVector<TCandidate> expected = { +            {ClusterName, "example"}, +            {ClusterName, "yt:saurus"}, +        }; +        auto engine = MakeSqlCompletionEngineUT(); +        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "USE "), expected);      }      Y_UNIT_TEST(Alter) { @@ -187,6 +237,28 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE "), expected);      } +    Y_UNIT_TEST(CreateTable) { +        auto engine = MakeSqlCompletionEngineUT(); +        { +            TVector<TCandidate> expected = { +                {FolderName, "`.sys/`"}, +                {FolderName, "`local/`"}, +                {FolderName, "`prod/`"}, +                {FolderName, "`test/`"}, +                {ClusterName, "example"}, +                {ClusterName, "yt:saurus"}, +                {Keyword, "IF NOT EXISTS"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE TABLE #"), expected); +        } +        { +            TVector<TCandidate> expected = { +                {FolderName, "service/"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE TABLE `test/#`"), expected); +        } +    } +      Y_UNIT_TEST(Delete) {          TVector<TCandidate> expected = {              {Keyword, "FROM"}, @@ -216,6 +288,21 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "DROP "), expected);      } +    Y_UNIT_TEST(DropObject) { +        TVector<TCandidate> expected = { +            {FolderName, "`.sys/`"}, +            {FolderName, "`local/`"}, +            {FolderName, "`prod/`"}, +            {FolderName, "`test/`"}, +            {ClusterName, "example"}, +            {ClusterName, "yt:saurus"}, +            {Keyword, "IF EXISTS"}, +        }; +        auto engine = MakeSqlCompletionEngineUT(); +        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "DROP TABLE "), expected); +        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "DROP VIEW "), expected); +    } +      Y_UNIT_TEST(Explain) {          TVector<TCandidate> expected = {              {Keyword, "ALTER"}, @@ -299,7 +386,9 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          {              TVector<TCandidate> expected = {                  {Keyword, "ANSI"}, -                {PragmaName, "yson.CastToString"}}; +                {PragmaName, "yson.CastToString"}, +                {PragmaName, "yt.RuntimeCluster"}, +                {PragmaName, "yt.RuntimeClusterSelection"}};              auto completion = engine->CompleteAsync({"PRAGMA "}).GetValueSync();              UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);              UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, ""); @@ -332,6 +421,23 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {              UNIT_ASSERT_VALUES_EQUAL(completion.Candidates, expected);              UNIT_ASSERT_VALUES_EQUAL(completion.CompletedToken.Content, "cast");          } +        { +            TVector<TCandidate> expected = { +                {PragmaName, "RuntimeCluster"}, +                {PragmaName, "RuntimeClusterSelection"}}; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "pragma yt."), expected); +            UNIT_ASSERT_VALUES_EQUAL( +                Complete(engine, "pragma yt.RuntimeClusterSelection='force';\npragma yt.Ru"), +                expected); +        } +        { +            TVector<TCandidate> expected = { +                {PragmaName, "RuntimeCluster"}, +                {PragmaName, "RuntimeClusterSelection"}}; +            UNIT_ASSERT_VALUES_EQUAL( +                Complete(engine, "pragma yt.Ru#\n"), +                expected); +        }      }      Y_UNIT_TEST(Select) { @@ -377,23 +483,132 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {      }      Y_UNIT_TEST(SelectFrom) { -        TVector<TCandidate> expected = { -            {Keyword, "ANY"}, -            {Keyword, "CALLABLE"}, -            {Keyword, "DICT"}, -            {Keyword, "ENUM"}, -            {Keyword, "FLOW"}, -            {Keyword, "LIST"}, -            {Keyword, "OPTIONAL"}, -            {Keyword, "RESOURCE"}, -            {Keyword, "SET"}, -            {Keyword, "STRUCT"}, -            {Keyword, "TAGGED"}, -            {Keyword, "TUPLE"}, -            {Keyword, "VARIANT"}, -        };          auto engine = MakeSqlCompletionEngineUT(); -        UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM "), expected); +        { +            TVector<TCandidate> expected = { +                {FolderName, "`.sys/`"}, +                {FolderName, "`local/`"}, +                {FolderName, "`prod/`"}, +                {FolderName, "`test/`"}, +                {ClusterName, "example"}, +                {ClusterName, "yt:saurus"}, +                {Keyword, "ANY"}, +                {Keyword, "CALLABLE"}, +                {Keyword, "DICT"}, +                {Keyword, "ENUM"}, +                {Keyword, "FLOW"}, +                {Keyword, "LIST"}, +                {Keyword, "OPTIONAL"}, +                {Keyword, "RESOURCE"}, +                {Keyword, "SET"}, +                {Keyword, "STRUCT"}, +                {Keyword, "TAGGED"}, +                {Keyword, "TUPLE"}, +                {Keyword, "VARIANT"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM "), expected); +        } +        { +            TVector<TCandidate> expected = {}; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM `#"), expected); +        } +        { +            TString input = "SELECT * FROM `#`"; +            TVector<TCandidate> expected = { +                {FolderName, ".sys/"}, +                {FolderName, "local/"}, +                {FolderName, "prod/"}, +                {FolderName, "test/"}, +            }; +            TCompletion actual = engine->Complete(SharpedInput(input)); +            UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); +            UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, ""); +        } +        { +            TString input = "SELECT * FROM `local/#`"; +            TVector<TCandidate> expected = { +                {TableName, "abacaba"}, +                {TableName, "account"}, +                {TableName, "example"}, +            }; +            TCompletion actual = engine->Complete(SharpedInput(input)); +            UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); +            UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, ""); +        } +        { +            TString input = "SELECT * FROM `local/a#`"; +            TVector<TCandidate> expected = { +                {TableName, "abacaba"}, +                {TableName, "account"}, +            }; +            TCompletion actual = engine->Complete(SharpedInput(input)); +            UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); +            UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, "a"); +        } +        { +            TString input = "SELECT * FROM `.sy#`"; +            TVector<TCandidate> expected = { +                {FolderName, ".sys/"}, +            }; +            TCompletion actual = engine->Complete(SharpedInput(input)); +            UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); +            UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, ".sy"); +        } +        { +            TString input = "SELECT * FROM `/test/ser#vice/`"; +            TVector<TCandidate> expected = { +                {FolderName, "service/"}, +            }; +            TCompletion actual = engine->Complete(SharpedInput(input)); +            UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); +            UNIT_ASSERT_VALUES_EQUAL(actual.CompletedToken.Content, "ser"); +        } +    } + +    Y_UNIT_TEST(SelectFromCluster) { +        auto engine = MakeSqlCompletionEngineUT(); +        { +            TVector<TCandidate> expected = { +                {ClusterName, "yt:saurus"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt#"), expected); +        } +        { +            TVector<TCandidate> expected = { +                {ClusterName, "saurus"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:"), expected); +        } +        { +            TVector<TCandidate> expected = { +                {ClusterName, "saurus"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:saurus#"), expected); +        } +        { +            TVector<TCandidate> expected = { +                {TableName, "`maxim`"}, +                {Keyword, "CALLABLE"}, +                {Keyword, "DICT"}, +                {Keyword, "ENUM"}, +                {Keyword, "FLOW"}, +                {Keyword, "LIST"}, +                {Keyword, "OPTIONAL"}, +                {Keyword, "RESOURCE"}, +                {Keyword, "SET"}, +                {Keyword, "STRUCT"}, +                {Keyword, "TAGGED"}, +                {Keyword, "TUPLE"}, +                {Keyword, "VARIANT"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:saurus."), expected); +        } +        { +            TVector<TCandidate> expected = { +                {TableName, "`people`"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(CompleteTop(1, engine, "SELECT * FROM example."), expected); +        }      }      Y_UNIT_TEST(SelectWhere) { @@ -445,6 +660,28 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT "), expected);      } +    Y_UNIT_TEST(UpsertInto) { +        auto engine = MakeSqlCompletionEngineUT(); +        { +            TVector<TCandidate> expected = { +                {FolderName, "`.sys/`"}, +                {FolderName, "`local/`"}, +                {FolderName, "`prod/`"}, +                {FolderName, "`test/`"}, +                {ClusterName, "example"}, +                {ClusterName, "yt:saurus"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT INTO "), expected); +        } +        { +            TVector<TCandidate> expected = { +                {TableName, "meta"}, +                {FolderName, "service/"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT INTO `test/#`"), expected); +        } +    } +      Y_UNIT_TEST(TypeName) {          TVector<TCandidate> expected = {              {Keyword, "CALLABLE<("}, @@ -559,6 +796,52 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "INSERT INTO my_table WITH "), expected);      } +    Y_UNIT_TEST(CursorPosition) { +        auto engine = MakeSqlCompletionEngineUT(); +        { +            TVector<TCandidate> expected = { +                {Keyword, "AND"}, +                {Keyword, "AS"}, +                {Keyword, "ASSUME"}, +                {Keyword, "BETWEEN"}, +                {Keyword, "COLLATE"}, +                {Keyword, "EXCEPT"}, +                {Keyword, "FROM"}, +                {Keyword, "GLOB"}, +                {Keyword, "GROUP"}, +                {Keyword, "HAVING"}, +                {Keyword, "ILIKE"}, +                {Keyword, "IN"}, +                {Keyword, "INTERSECT"}, +                {Keyword, "INTO RESULT"}, +                {Keyword, "IS"}, +                {Keyword, "ISNULL"}, +                {Keyword, "LIKE"}, +                {Keyword, "LIMIT"}, +                {Keyword, "MATCH"}, +                {Keyword, "NOT"}, +                {Keyword, "NOTNULL"}, +                {Keyword, "OR"}, +                {Keyword, "ORDER BY"}, +                {Keyword, "REGEXP"}, +                {Keyword, "RLIKE"}, +                {Keyword, "UNION"}, +                {Keyword, "WHERE"}, +                {Keyword, "WINDOW"}, +                {Keyword, "WITHOUT"}, +                {Keyword, "XOR"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT `a`"), expected); +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT `a`#FROM"), expected); +        } +        { +            TVector<TCandidate> expected = { +                {Keyword, "FROM"}, +            }; +            UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM# "), expected); +        } +    } +      Y_UNIT_TEST(Enclosed) {          TVector<TCandidate> empty = {}; @@ -634,11 +917,11 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) {          wchar32 rune;          while (ptr < end) {              Y_ENSURE(ReadUTF8CharAndAdvance(rune, ptr, end) == RECODE_OK); -            TCompletion completion = engine->CompleteAsync({ -                                                               .Text = query, -                                                               .CursorPosition = static_cast<size_t>(std::distance(begin, ptr)), -                                                           }) -                                         .GetValueSync(); +            TCompletionInput input = { +                .Text = query, +                .CursorPosition = static_cast<size_t>(std::distance(begin, ptr)), +            }; +            TCompletion completion = engine->CompleteAsync(input).GetValueSync();              Y_DO_NOT_OPTIMIZE_AWAY(completion);          }      } diff --git a/yql/essentials/sql/v1/complete/syntax/cursor_token_context.cpp b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.cpp new file mode 100644 index 00000000000..33aef36847a --- /dev/null +++ b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.cpp @@ -0,0 +1,160 @@ +#include "cursor_token_context.h" + +#include <yql/essentials/core/issue/yql_issue.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> + +namespace NSQLComplete { + +    namespace { + +        bool Tokenize(ILexer::TPtr& lexer, TCompletionInput input, TParsedTokenList& tokens) { +            NYql::TIssues issues; +            if (!NSQLTranslation::Tokenize( +                    *lexer, TString(input.Text), /* queryName = */ "", +                    tokens, issues, /* maxErrors = */ 1)) { +                return false; +            } +            return true; +        } + +        TCursor GetCursor(const TParsedTokenList& tokens, size_t cursorPosition) { +            size_t current = 0; +            for (size_t i = 0; i < tokens.size() && current < cursorPosition; ++i) { +                const auto& content = tokens[i].Content; + +                current += content.size(); +                if (current < cursorPosition) { +                    continue; +                } + +                TCursor cursor = { +                    .PrevTokenIndex = i, +                    .NextTokenIndex = i, +                    .Position = cursorPosition, +                }; + +                if (current == cursorPosition) { +                    cursor.NextTokenIndex += 1; +                } + +                return cursor; +            } + +            return { +                .PrevTokenIndex = Nothing(), +                .NextTokenIndex = 0, +                .Position = cursorPosition, +            }; +        } + +        TVector<size_t> GetTokenPositions(const TParsedTokenList& tokens) { +            TVector<size_t> positions; +            positions.reserve(tokens.size()); +            size_t pos = 0; +            for (const auto& token : tokens) { +                positions.emplace_back(pos); +                pos += token.Content.size(); +            } +            return positions; +        } + +    } // namespace + +    bool TRichParsedToken::IsLiteral() const { +        return Base->Name == "STRING_VALUE" || +               Base->Name == "DIGIGTS" || +               Base->Name == "INTEGER_VALUE" || +               Base->Name == "REAL"; +    } + +    TRichParsedToken TokenAt(const TCursorTokenContext& context, size_t index) { +        return { +            .Base = &context.Tokens.at(index), +            .Index = index, +            .Position = context.TokenPositions.at(index), +        }; +    } + +    TMaybe<TRichParsedToken> TCursorTokenContext::Enclosing() const { +        if (Tokens.size() == 1) { +            Y_ENSURE(Tokens[0].Name == "EOF"); +            return Nothing(); +        } + +        if (Cursor.PrevTokenIndex.Empty()) { +            return Nothing(); +        } + +        auto token = TokenAt(*this, *Cursor.PrevTokenIndex); +        if (Cursor.PrevTokenIndex == Cursor.NextTokenIndex || +            !IsWordBoundary(token.Base->Content.back())) { +            return token; +        } + +        return Nothing(); +    } + +    TMaybe<TRichParsedToken> TCursorTokenContext::MatchCursorPrefix(const TVector<TStringBuf>& pattern) const { +        const auto prefix = std::span{Tokens.begin(), Cursor.NextTokenIndex}; +        if (prefix.size() < pattern.size()) { +            return Nothing(); +        } + +        ssize_t i = static_cast<ssize_t>(prefix.size()) - 1; +        ssize_t j = static_cast<ssize_t>(pattern.size()) - 1; +        for (; 0 <= j; --i, --j) { +            if (!pattern[j].empty() && prefix[i].Name != pattern[j]) { +                return Nothing(); +            } +        } +        return TokenAt(*this, prefix.size() - pattern.size()); +    } + +    bool GetStatement( +        ILexer::TPtr& lexer, +        TCompletionInput input, +        TCompletionInput& output, +        size_t& output_position) { +        TVector<TString> statements; +        NYql::TIssues issues; +        if (!NSQLTranslationV1::SplitQueryToStatements( +                TString(input.Text) + ";", lexer, +                statements, issues, /* file = */ "", +                /* areBlankSkipped = */ false)) { +            return false; +        } + +        size_t& cursor = output_position; +        cursor = 0; +        for (const auto& statement : statements) { +            if (input.CursorPosition < cursor + statement.size()) { +                output = { +                    .Text = input.Text.SubStr(cursor, statement.size()), +                    .CursorPosition = input.CursorPosition - cursor, +                }; +                return true; +            } +            cursor += statement.size(); +        } + +        output = input; +        return true; +    } + +    bool GetCursorTokenContext(ILexer::TPtr& lexer, TCompletionInput input, TCursorTokenContext& context) { +        TParsedTokenList tokens; +        if (!Tokenize(lexer, input, tokens)) { +            return false; +        } + +        TVector<size_t> positions = GetTokenPositions(tokens); +        TCursor cursor = GetCursor(tokens, input.CursorPosition); +        context = { +            .Tokens = std::move(tokens), +            .TokenPositions = std::move(positions), +            .Cursor = cursor, +        }; +        return true; +    } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/cursor_token_context.h b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.h new file mode 100644 index 00000000000..35d22231e35 --- /dev/null +++ b/yql/essentials/sql/v1/complete/syntax/cursor_token_context.h @@ -0,0 +1,50 @@ +#pragma once + +#include <yql/essentials/sql/v1/complete/core/input.h> +#include <yql/essentials/sql/v1/complete/text/word.h> + +#include <yql/essentials/parser/lexer_common/lexer.h> + +#include <util/generic/maybe.h> + +namespace NSQLComplete { + +    using NSQLTranslation::ILexer; +    using NSQLTranslation::TParsedToken; +    using NSQLTranslation::TParsedTokenList; + +    struct TCursor { +        TMaybe<size_t> PrevTokenIndex = Nothing(); +        size_t NextTokenIndex = PrevTokenIndex ? *PrevTokenIndex : 0; +        size_t Position = 0; +    }; + +    struct TRichParsedToken { +        const TParsedToken* Base = nullptr; +        size_t Index = 0; +        size_t Position = 0; + +        bool IsLiteral() const; +    }; + +    struct TCursorTokenContext { +        TParsedTokenList Tokens; +        TVector<size_t> TokenPositions; +        TCursor Cursor; + +        TMaybe<TRichParsedToken> Enclosing() const; +        TMaybe<TRichParsedToken> MatchCursorPrefix(const TVector<TStringBuf>& pattern) const; +    }; + +    bool GetStatement( +        ILexer::TPtr& lexer, +        TCompletionInput input, +        TCompletionInput& output, +        size_t& output_position); + +    bool GetCursorTokenContext( +        ILexer::TPtr& lexer, +        TCompletionInput input, +        TCursorTokenContext& context); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/cursor_token_context_ut.cpp b/yql/essentials/sql/v1/complete/syntax/cursor_token_context_ut.cpp new file mode 100644 index 00000000000..0e275cca3b8 --- /dev/null +++ b/yql/essentials/sql/v1/complete/syntax/cursor_token_context_ut.cpp @@ -0,0 +1,50 @@ +#include "cursor_token_context.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <yql/essentials/sql/v1/lexer/antlr4_pure/lexer.h> +#include <yql/essentials/sql/v1/lexer/lexer.h> + +using namespace NSQLComplete; + +Y_UNIT_TEST_SUITE(CursorTokenContextTests) { + +    NSQLTranslation::ILexer::TPtr MakeLexer() { +        NSQLTranslationV1::TLexers lexers; +        lexers.Antlr4Pure = NSQLTranslationV1::MakeAntlr4PureLexerFactory(); +        return NSQLTranslationV1::MakeLexer( +            lexers, /* ansi = */ false, /* antlr4 = */ true, +            NSQLTranslationV1::ELexerFlavor::Pure); +    } + +    TCursorTokenContext Context(TString input) { +        auto lexer = MakeLexer(); +        TCursorTokenContext context; +        UNIT_ASSERT(GetCursorTokenContext(lexer, SharpedInput(input), context)); +        return context; +    } + +    Y_UNIT_TEST(Empty) { +        auto context = Context(""); +        UNIT_ASSERT(context.Cursor.PrevTokenIndex.Empty()); +        UNIT_ASSERT_VALUES_EQUAL(context.Cursor.NextTokenIndex, 0); +        UNIT_ASSERT_VALUES_EQUAL(context.Cursor.Position, 0); +        UNIT_ASSERT(context.Enclosing().Empty()); +    } + +    Y_UNIT_TEST(Blank) { +        UNIT_ASSERT(Context("# ").Enclosing().Empty()); +        UNIT_ASSERT(Context(" #").Enclosing().Empty()); +        UNIT_ASSERT(Context(" # ").Enclosing().Empty()); +    } + +    Y_UNIT_TEST(Enclosing) { +        UNIT_ASSERT(Context("se#").Enclosing().Defined()); +        UNIT_ASSERT(Context("#se").Enclosing().Empty()); +        UNIT_ASSERT(Context("`se`#").Enclosing().Empty()); +        UNIT_ASSERT(Context("#`se`").Enclosing().Empty()); +        UNIT_ASSERT(Context("`se`#`se`").Enclosing().Defined()); +        UNIT_ASSERT(Context("\"se\"#\"se\"").Enclosing().Empty()); +    } + +} // Y_UNIT_TEST_SUITE(CursorTokenContextTests) diff --git a/yql/essentials/sql/v1/complete/syntax/format.cpp b/yql/essentials/sql/v1/complete/syntax/format.cpp index 1c9f146c923..43c36aea9dd 100644 --- a/yql/essentials/sql/v1/complete/syntax/format.cpp +++ b/yql/essentials/sql/v1/complete/syntax/format.cpp @@ -35,4 +35,17 @@ namespace NSQLComplete {          return text;      } +    TString Quoted(TString content) { +        content.prepend('`'); +        content.append('`'); +        return content; +    } + +    TString Unquoted(TString content) { +        Y_ENSURE(2 <= content.size() && content.front() == '`' && content.back() == '`'); +        content.erase(0, 1); +        content.pop_back(); +        return content; +    } +  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/format.h b/yql/essentials/sql/v1/complete/syntax/format.h index 6c2f1b72ac2..58e5d1f1e4a 100644 --- a/yql/essentials/sql/v1/complete/syntax/format.h +++ b/yql/essentials/sql/v1/complete/syntax/format.h @@ -6,5 +6,7 @@  namespace NSQLComplete {      TString FormatKeywords(const TVector<TString>& seq); +    TString Quoted(TString content); +    TString Unquoted(TString content);  } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/grammar.cpp b/yql/essentials/sql/v1/complete/syntax/grammar.cpp index 252deaf682c..c080fae5ae4 100644 --- a/yql/essentials/sql/v1/complete/syntax/grammar.cpp +++ b/yql/essentials/sql/v1/complete/syntax/grammar.cpp @@ -7,31 +7,31 @@ namespace NSQLComplete {      class TSqlGrammar: public ISqlGrammar {      public:          TSqlGrammar(const NSQLReflect::TLexerGrammar& grammar) -            : Parser(MakeDummyParser()) -            , AllTokens(ComputeAllTokens()) -            , KeywordTokens(ComputeKeywordTokens(grammar)) -            , PunctuationTokens(ComputePunctuationTokens(grammar)) +            : Parser_(MakeDummyParser()) +            , AllTokens_(ComputeAllTokens()) +            , KeywordTokens_(ComputeKeywordTokens(grammar)) +            , PunctuationTokens_(ComputePunctuationTokens(grammar))          {          }          const antlr4::dfa::Vocabulary& GetVocabulary() const override { -            return Parser->getVocabulary(); +            return Parser_->getVocabulary();          }          const std::unordered_set<TTokenId>& GetAllTokens() const override { -            return AllTokens; +            return AllTokens_;          }          const std::unordered_set<TTokenId>& GetKeywordTokens() const override { -            return KeywordTokens; +            return KeywordTokens_;          }          const std::unordered_set<TTokenId>& GetPunctuationTokens() const override { -            return PunctuationTokens; +            return PunctuationTokens_;          }          const std::string& SymbolizedRule(TRuleId rule) const override { -            return Parser->getRuleNames().at(rule); +            return Parser_->getRuleNames().at(rule);          }      private: @@ -76,10 +76,10 @@ namespace NSQLComplete {              return punctuationTokens;          } -        const THolder<antlr4::Parser> Parser; -        const std::unordered_set<TTokenId> AllTokens; -        const std::unordered_set<TTokenId> KeywordTokens; -        const std::unordered_set<TTokenId> PunctuationTokens; +        const THolder<antlr4::Parser> Parser_; +        const std::unordered_set<TTokenId> AllTokens_; +        const std::unordered_set<TTokenId> KeywordTokens_; +        const std::unordered_set<TTokenId> PunctuationTokens_;      };      const ISqlGrammar& GetSqlGrammar() { diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp index c434fa28daf..549208d4cab 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.cpp +++ b/yql/essentials/sql/v1/complete/syntax/local.cpp @@ -1,9 +1,10 @@  #include "local.h"  #include "ansi.h" +#include "cursor_token_context.h" +#include "format.h"  #include "grammar.h"  #include "parser_call_stack.h" -#include "token.h"  #include <yql/essentials/sql/v1/complete/antlr4/c3i.h>  #include <yql/essentials/sql/v1/complete/antlr4/c3t.h> @@ -49,65 +50,77 @@ namespace NSQLComplete {      public:          explicit TSpecializedLocalSyntaxAnalysis(TLexerSupplier lexer) -            : Grammar(&GetSqlGrammar()) +            : Grammar_(&GetSqlGrammar())              , Lexer_(lexer(/* ansi = */ IsAnsiLexer)) -            , C3(ComputeC3Config()) +            , C3_(ComputeC3Config())          {          }          TLocalSyntaxContext Analyze(TCompletionInput input) override {              TCompletionInput statement; -            if (!GetStatement(Lexer_, input, statement)) { +            size_t statement_position; +            if (!GetStatement(Lexer_, input, statement, statement_position)) {                  return {};              } -            auto candidates = C3.Complete(statement); - -            TParsedTokenList tokens; -            TCaretTokenPosition caret; -            if (!TokenizePrefix(statement, tokens, caret)) { +            TCursorTokenContext context; +            if (!GetCursorTokenContext(Lexer_, statement, context)) {                  return {};              } -            if (IsCaretEnslosed(tokens, caret)) { -                return {}; +            TC3Candidates candidates = C3_.Complete(statement); + +            TLocalSyntaxContext result; + +            result.EditRange = EditRange(context); +            result.EditRange.Begin += statement_position; + +            if (auto enclosing = context.Enclosing()) { +                if (enclosing->IsLiteral()) { +                    return result; +                } else if (enclosing->Base->Name == "ID_QUOTED") { +                    result.Object = ObjectMatch(context, candidates); +                    return result; +                }              } -            return { -                .Keywords = SiftedKeywords(candidates), -                .Pragma = PragmaMatch(tokens, candidates), -                .IsTypeName = IsTypeNameMatched(candidates), -                .Function = FunctionMatch(tokens, candidates), -                .Hint = HintMatch(candidates), -            }; +            result.Keywords = SiftedKeywords(candidates); +            result.Pragma = PragmaMatch(context, candidates); +            result.Type = TypeMatch(candidates); +            result.Function = FunctionMatch(context, candidates); +            result.Hint = HintMatch(candidates); +            result.Object = ObjectMatch(context, candidates); +            result.Cluster = ClusterMatch(context, candidates); + +            return result;          }      private: -        IC3Engine::TConfig ComputeC3Config() { +        IC3Engine::TConfig ComputeC3Config() const {              return {                  .IgnoredTokens = ComputeIgnoredTokens(),                  .PreferredRules = ComputePreferredRules(),              };          } -        std::unordered_set<TTokenId> ComputeIgnoredTokens() { -            auto ignoredTokens = Grammar->GetAllTokens(); -            for (auto keywordToken : Grammar->GetKeywordTokens()) { +        std::unordered_set<TTokenId> ComputeIgnoredTokens() const { +            auto ignoredTokens = Grammar_->GetAllTokens(); +            for (auto keywordToken : Grammar_->GetKeywordTokens()) {                  ignoredTokens.erase(keywordToken);              } -            for (auto punctuationToken : Grammar->GetPunctuationTokens()) { +            for (auto punctuationToken : Grammar_->GetPunctuationTokens()) {                  ignoredTokens.erase(punctuationToken);              }              return ignoredTokens;          } -        std::unordered_set<TRuleId> ComputePreferredRules() { +        std::unordered_set<TRuleId> ComputePreferredRules() const {              return GetC3PreferredRules();          } -        TLocalSyntaxContext::TKeywords SiftedKeywords(const TC3Candidates& candidates) { -            const auto& vocabulary = Grammar->GetVocabulary(); -            const auto& keywordTokens = Grammar->GetKeywordTokens(); +        TLocalSyntaxContext::TKeywords SiftedKeywords(const TC3Candidates& candidates) const { +            const auto& vocabulary = Grammar_->GetVocabulary(); +            const auto& keywordTokens = Grammar_->GetKeywordTokens();              TLocalSyntaxContext::TKeywords keywords;              for (const auto& token : candidates.Tokens) { @@ -122,40 +135,41 @@ namespace NSQLComplete {          }          TMaybe<TLocalSyntaxContext::TPragma> PragmaMatch( -            const TParsedTokenList& tokens, const TC3Candidates& candidates) { +            const TCursorTokenContext& context, const TC3Candidates& candidates) const {              if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyPragmaStack))) {                  return Nothing();              }              TLocalSyntaxContext::TPragma pragma; -            if (EndsWith(tokens, {"ID_PLAIN", "DOT"})) { -                pragma.Namespace = tokens[tokens.size() - 2].Content; -            } else if (EndsWith(tokens, {"ID_PLAIN", "DOT", ""})) { -                pragma.Namespace = tokens[tokens.size() - 3].Content; + +            if (TMaybe<TRichParsedToken> begin; +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT"})) || +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT", ""}))) { +                pragma.Namespace = begin->Base->Content;              }              return pragma;          } -        bool IsTypeNameMatched(const TC3Candidates& candidates) { +        bool TypeMatch(const TC3Candidates& candidates) const {              return AnyOf(candidates.Rules, RuleAdapted(IsLikelyTypeStack));          }          TMaybe<TLocalSyntaxContext::TFunction> FunctionMatch( -            const TParsedTokenList& tokens, const TC3Candidates& candidates) { +            const TCursorTokenContext& context, const TC3Candidates& candidates) const {              if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyFunctionStack))) {                  return Nothing();              }              TLocalSyntaxContext::TFunction function; -            if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE"})) { -                function.Namespace = tokens[tokens.size() - 2].Content; -            } else if (EndsWith(tokens, {"ID_PLAIN", "NAMESPACE", ""})) { -                function.Namespace = tokens[tokens.size() - 3].Content; +            if (TMaybe<TRichParsedToken> begin; +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "NAMESPACE"})) || +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "NAMESPACE", ""}))) { +                function.Namespace = begin->Base->Content;              }              return function;          } -        TMaybe<TLocalSyntaxContext::THint> HintMatch(const TC3Candidates& candidates) { +        TMaybe<TLocalSyntaxContext::THint> HintMatch(const TC3Candidates& candidates) const {              // TODO(YQL-19747): detect local contexts with a single iteration through the candidates.Rules              auto rule = FindIf(candidates.Rules, RuleAdapted(IsLikelyHintStack));              if (rule == std::end(candidates.Rules)) { @@ -172,45 +186,103 @@ namespace NSQLComplete {              };          } -        bool TokenizePrefix(TCompletionInput input, TParsedTokenList& tokens, TCaretTokenPosition& caret) { -            NYql::TIssues issues; -            if (!NSQLTranslation::Tokenize( -                    *Lexer_, TString(input.Text), /* queryName = */ "", -                    tokens, issues, /* maxErrors = */ 1)) { -                return false; +        TMaybe<TLocalSyntaxContext::TObject> ObjectMatch( +            const TCursorTokenContext& context, const TC3Candidates& candidates) const { +            TLocalSyntaxContext::TObject object; + +            if (AnyOf(candidates.Rules, RuleAdapted(IsLikelyObjectRefStack))) { +                object.Kinds.emplace(EObjectKind::Folder); +            } + +            if (AnyOf(candidates.Rules, RuleAdapted(IsLikelyExistingTableStack))) { +                object.Kinds.emplace(EObjectKind::Folder); +                object.Kinds.emplace(EObjectKind::Table); +            } + +            if (object.Kinds.empty()) { +                return Nothing();              } -            Y_ENSURE(!tokens.empty() && tokens.back().Name == "EOF"); -            tokens.pop_back(); +            if (TMaybe<TRichParsedToken> begin; +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT"})) || +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "DOT", ""}))) { +                object.Cluster = begin->Base->Content; +            } -            caret = CaretTokenPosition(tokens, input.CursorPosition); -            tokens.crop(caret.NextTokenIndex + 1); -            return true; +            if (TMaybe<TRichParsedToken> begin; +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON", "ID_PLAIN", "DOT"})) || +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON", "ID_PLAIN", "DOT", ""}))) { +                object.Provider = begin->Base->Content; +            } + +            if (auto path = ObjectPath(context)) { +                object.Path = *path; +                object.IsEnclosed = true; +            } + +            return object; +        } + +        TMaybe<TString> ObjectPath(const TCursorTokenContext& context) const { +            if (auto enclosing = context.Enclosing()) { +                TString path = enclosing->Base->Content; +                if (enclosing->Base->Name == "ID_QUOTED") { +                    path = Unquoted(std::move(path)); +                } +                path.resize(context.Cursor.Position - enclosing->Position - 1); +                return path; +            } +            return Nothing();          } -        bool IsCaretEnslosed(const TParsedTokenList& tokens, TCaretTokenPosition caret) { -            if (tokens.empty() || caret.PrevTokenIndex != caret.NextTokenIndex) { -                return false; +        TMaybe<TLocalSyntaxContext::TCluster> ClusterMatch( +            const TCursorTokenContext& context, const TC3Candidates& candidates) const { +            if (!AnyOf(candidates.Rules, RuleAdapted(IsLikelyClusterStack))) { +                return Nothing();              } -            const auto& token = tokens.back(); -            return token.Name == "STRING_VALUE" || -                   token.Name == "ID_QUOTED" || -                   token.Name == "DIGIGTS" || -                   token.Name == "INTEGER_VALUE" || -                   token.Name == "REAL"; +            TLocalSyntaxContext::TCluster cluster; +            if (TMaybe<TRichParsedToken> begin; +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON"})) || +                (begin = context.MatchCursorPrefix({"ID_PLAIN", "COLON", ""}))) { +                cluster.Provider = begin->Base->Content; +            } +            return cluster; +        } + +        TEditRange EditRange(const TCursorTokenContext& context) const { +            if (auto enclosing = context.Enclosing()) { +                return EditRange(*enclosing, context.Cursor); +            } + +            return { +                .Begin = context.Cursor.Position, +                .Length = 0, +            }; +        } + +        TEditRange EditRange(const TRichParsedToken& token, const TCursor& cursor) const { +            size_t begin = token.Position; +            if (token.Base->Name == "NOT_EQUALS2") { +                begin += 1; +            } + +            return { +                .Begin = begin, +                .Length = cursor.Position - begin, +            };          } -        const ISqlGrammar* Grammar; +        const ISqlGrammar* Grammar_;          NSQLTranslation::ILexer::TPtr Lexer_; -        TC3Engine<G> C3; +        TC3Engine<G> C3_;      };      class TLocalSyntaxAnalysis: public ILocalSyntaxAnalysis {      public:          explicit TLocalSyntaxAnalysis(TLexerSupplier lexer) -            : DefaultEngine(lexer) -            , AnsiEngine(lexer) +            : DefaultEngine_(lexer) +            , AnsiEngine_(lexer)          {          } @@ -223,13 +295,13 @@ namespace NSQLComplete {      private:          ILocalSyntaxAnalysis& GetSpecializedEngine(bool isAnsiLexer) {              if (isAnsiLexer) { -                return AnsiEngine; +                return AnsiEngine_;              } -            return DefaultEngine; +            return DefaultEngine_;          } -        TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ false> DefaultEngine; -        TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ true> AnsiEngine; +        TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ false> DefaultEngine_; +        TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ true> AnsiEngine_;      };      ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis(TLexerSupplier lexer) { diff --git a/yql/essentials/sql/v1/complete/syntax/local.h b/yql/essentials/sql/v1/complete/syntax/local.h index d58b62c62cd..8f88d5aa71c 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.h +++ b/yql/essentials/sql/v1/complete/syntax/local.h @@ -1,15 +1,22 @@  #pragma once +#include <yql/essentials/sql/v1/complete/core/name.h>  #include <yql/essentials/sql/v1/complete/sql_complete.h>  #include <yql/essentials/sql/v1/lexer/lexer.h>  #include <util/generic/string.h>  #include <util/generic/hash.h> +#include <util/generic/hash_set.h>  #include <util/generic/maybe.h>  namespace NSQLComplete { +    struct TEditRange { +        size_t Begin = 0; +        size_t Length = 0; +    }; +      struct TLocalSyntaxContext {          using TKeywords = THashMap<TString, TVector<TString>>; @@ -25,11 +32,26 @@ namespace NSQLComplete {              EStatementKind StatementKind;          }; +        struct TCluster { +            TString Provider; +        }; + +        struct TObject { +            TString Provider; +            TString Cluster; +            TString Path; +            THashSet<EObjectKind> Kinds; +            bool IsEnclosed = false; +        }; +          TKeywords Keywords;          TMaybe<TPragma> Pragma; -        bool IsTypeName = false; +        bool Type = false;          TMaybe<TFunction> Function;          TMaybe<THint> Hint; +        TMaybe<TObject> Object; +        TMaybe<TCluster> Cluster; +        TEditRange EditRange;      };      class ILocalSyntaxAnalysis { diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp index 938483438b1..ce6c94306d4 100644 --- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp +++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.cpp @@ -13,7 +13,7 @@  namespace NSQLComplete { -    const TVector<TRuleId> KeywordRules = { +    const TVector<TRuleId> PreferredRules = {          RULE(Keyword),          RULE(Keyword_expr_uncompat),          RULE(Keyword_table_uncompat), @@ -24,27 +24,13 @@ namespace NSQLComplete {          RULE(Keyword_hint_uncompat),          RULE(Keyword_as_compat),          RULE(Keyword_compat), -    }; - -    const TVector<TRuleId> PragmaNameRules = { -        RULE(Opt_id_prefix_or_type), -        RULE(An_id), -    }; - -    const TVector<TRuleId> TypeNameRules = { -        RULE(Type_name_simple),          RULE(An_id_or_type), -    }; - -    const TVector<TRuleId> FunctionNameRules = { +        RULE(An_id),          RULE(Id_expr), -        RULE(An_id_or_type),          RULE(Id_or_type), -    }; - -    const TVector<TRuleId> HintNameRules = {          RULE(Id_hint), -        RULE(An_id), +        RULE(Opt_id_prefix_or_type), +        RULE(Type_name_simple),      };      TVector<std::string> Symbolized(const TParserCallStack& stack) { @@ -101,6 +87,26 @@ namespace NSQLComplete {                 Contains({RULE(External_call_param), RULE(An_id)}, stack);      } +    bool IsLikelyObjectRefStack(const TParserCallStack& stack) { +        return Contains({RULE(Object_ref)}, stack); +    } + +    bool IsLikelyExistingTableStack(const TParserCallStack& stack) { +        return !Contains({RULE(Create_table_stmt), +                          RULE(Simple_table_ref)}, stack) && +               (Contains({RULE(Simple_table_ref), +                          RULE(Simple_table_ref_core), +                          RULE(Object_ref)}, stack) || +                Contains({RULE(Single_source), +                          RULE(Table_ref), +                          RULE(Table_key), +                          RULE(Id_table_or_type)}, stack)); +    } + +    bool IsLikelyClusterStack(const TParserCallStack& stack) { +        return Contains({RULE(Cluster_expr)}, stack); +    } +      TMaybe<EStatementKind> StatementKindOf(const TParserCallStack& stack) {          for (TRuleId rule : std::ranges::views::reverse(stack)) {              if (rule == RULE(Process_core) || rule == RULE(Reduce_core) || rule == RULE(Select_core)) { @@ -115,10 +121,7 @@ namespace NSQLComplete {      std::unordered_set<TRuleId> GetC3PreferredRules() {          std::unordered_set<TRuleId> preferredRules; -        preferredRules.insert(std::begin(KeywordRules), std::end(KeywordRules)); -        preferredRules.insert(std::begin(PragmaNameRules), std::end(PragmaNameRules)); -        preferredRules.insert(std::begin(TypeNameRules), std::end(TypeNameRules)); -        preferredRules.insert(std::begin(FunctionNameRules), std::end(FunctionNameRules)); +        preferredRules.insert(std::begin(PreferredRules), std::end(PreferredRules));          return preferredRules;      } diff --git a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h index d185b72d628..d44b824a05e 100644 --- a/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h +++ b/yql/essentials/sql/v1/complete/syntax/parser_call_stack.h @@ -15,6 +15,12 @@ namespace NSQLComplete {      bool IsLikelyHintStack(const TParserCallStack& stack); +    bool IsLikelyObjectRefStack(const TParserCallStack& stack); + +    bool IsLikelyExistingTableStack(const TParserCallStack& stack); + +    bool IsLikelyClusterStack(const TParserCallStack& stack); +      TMaybe<EStatementKind> StatementKindOf(const TParserCallStack& stack);      std::unordered_set<TRuleId> GetC3PreferredRules(); diff --git a/yql/essentials/sql/v1/complete/syntax/token.cpp b/yql/essentials/sql/v1/complete/syntax/token.cpp deleted file mode 100644 index b8aee3211c6..00000000000 --- a/yql/essentials/sql/v1/complete/syntax/token.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "token.h" - -#include <yql/essentials/core/issue/yql_issue.h> -#include <yql/essentials/sql/v1/lexer/lexer.h> - -namespace NSQLComplete { - -    bool GetStatement(NSQLTranslation::ILexer::TPtr& lexer, TCompletionInput input, TCompletionInput& output) { -        TVector<TString> statements; -        NYql::TIssues issues; -        if (!NSQLTranslationV1::SplitQueryToStatements( -                TString(input.Text) + ";", lexer, -                statements, issues, /* file = */ "", -                /* areBlankSkipped = */ false)) { -            return false; -        } - -        size_t cursor = 0; -        for (const auto& statement : statements) { -            if (input.CursorPosition < cursor + statement.size()) { -                output = { -                    .Text = input.Text.SubStr(cursor, statement.size()), -                    .CursorPosition = input.CursorPosition - cursor, -                }; -                return true; -            } -            cursor += statement.size(); -        } - -        output = input; -        return true; -    } - -    TCaretTokenPosition CaretTokenPosition(const TParsedTokenList& tokens, size_t cursorPosition) { -        size_t cursor = 0; -        for (size_t i = 0; i < tokens.size(); ++i) { -            const auto& content = tokens[i].Content; -            cursor += content.size(); -            if (cursorPosition < cursor) { -                return {i, i}; -            } else if (cursorPosition == cursor && IsWordBoundary(content.back())) { -                return {i, i + 1}; -            } -        } -        return {std::max(tokens.size(), static_cast<size_t>(1)) - 1, tokens.size()}; -    } - -    bool EndsWith(const TParsedTokenList& tokens, const TVector<TStringBuf>& pattern) { -        if (tokens.size() < pattern.size()) { -            return false; -        } -        for (yssize_t i = tokens.ysize() - 1, j = pattern.ysize() - 1; 0 <= j; --i, --j) { -            if (!pattern[j].empty() && tokens[i].Name != pattern[j]) { -                return false; -            } -        } -        return true; -    } - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/token.h b/yql/essentials/sql/v1/complete/syntax/token.h deleted file mode 100644 index d1e215285a9..00000000000 --- a/yql/essentials/sql/v1/complete/syntax/token.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include <yql/essentials/sql/v1/complete/core/input.h> -#include <yql/essentials/sql/v1/complete/text/word.h> - -#include <yql/essentials/parser/lexer_common/lexer.h> - -namespace NSQLComplete { - -    using NSQLTranslation::TParsedTokenList; - -    // `PrevTokenIndex` = `NextTokenIndex`, iff caret is enclosed -    struct TCaretTokenPosition { -        size_t PrevTokenIndex; -        size_t NextTokenIndex; -    }; - -    bool GetStatement(NSQLTranslation::ILexer::TPtr& lexer, TCompletionInput input, TCompletionInput& output); - -    TCaretTokenPosition CaretTokenPosition(const TParsedTokenList& tokens, size_t cursorPosition); - -    bool EndsWith(const TParsedTokenList& tokens, const TVector<TStringBuf>& pattern); - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/ut/ya.make b/yql/essentials/sql/v1/complete/syntax/ut/ya.make index e070185af9f..7e682c5bac0 100644 --- a/yql/essentials/sql/v1/complete/syntax/ut/ya.make +++ b/yql/essentials/sql/v1/complete/syntax/ut/ya.make @@ -2,6 +2,11 @@ UNITTEST_FOR(yql/essentials/sql/v1/complete/syntax)  SRCS(      grammar_ut.cpp +    cursor_token_context_ut.cpp +) + +PEERDIR( +    yql/essentials/sql/v1/lexer/antlr4_pure  )  END() diff --git a/yql/essentials/sql/v1/complete/syntax/ya.make b/yql/essentials/sql/v1/complete/syntax/ya.make index 9e2e908454b..7f63e5b2374 100644 --- a/yql/essentials/sql/v1/complete/syntax/ya.make +++ b/yql/essentials/sql/v1/complete/syntax/ya.make @@ -2,11 +2,11 @@ LIBRARY()  SRCS(      ansi.cpp +    cursor_token_context.cpp      format.cpp      grammar.cpp      local.cpp      parser_call_stack.cpp -    token.cpp  )  ADDINCL( @@ -21,6 +21,8 @@ PEERDIR(      yql/essentials/sql/settings      yql/essentials/sql/v1/lexer      yql/essentials/sql/v1/reflect +    yql/essentials/sql/v1/complete/core +    yql/essentials/sql/v1/complete/text  )  END() diff --git a/yql/essentials/sql/v1/complete/ut/ya.make b/yql/essentials/sql/v1/complete/ut/ya.make index fbb84f56f25..c978e6e6048 100644 --- a/yql/essentials/sql/v1/complete/ut/ya.make +++ b/yql/essentials/sql/v1/complete/ut/ya.make @@ -7,7 +7,14 @@ SRCS(  PEERDIR(      yql/essentials/sql/v1/lexer/antlr4_pure      yql/essentials/sql/v1/lexer/antlr4_pure_ansi +    yql/essentials/sql/v1/complete/name/cluster/static +    yql/essentials/sql/v1/complete/name/object/dispatch +    yql/essentials/sql/v1/complete/name/object/simple +    yql/essentials/sql/v1/complete/name/object/simple/static +    yql/essentials/sql/v1/complete/name/service/cluster +    yql/essentials/sql/v1/complete/name/service/schema      yql/essentials/sql/v1/complete/name/service/static +    yql/essentials/sql/v1/complete/name/service/union  )  END()  | 
