From 815bb53e8a110e23218e610caabd6a3e15fa3ca5 Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Fri, 20 Jun 2025 11:33:48 +0300 Subject: Intermediate changes commit_hash:67576d4a9e4a1f4bbbb204bcb0b98c255009b1da --- .../cfg/tests/gateways-experimental.conf | 3 +- .../sql/v1/complete/name/cache/byte_size.h | 9 +++++ .../v1/complete/name/object/simple/cached/key.h | 27 ++++++++----- .../complete/name/object/simple/cached/schema.cpp | 31 +++++++++++---- .../v1/complete/name/object/simple/cached/schema.h | 12 +++++- .../complete/name/object/simple/static/schema.cpp | 15 ++++--- yql/essentials/sql/v1/complete/sql_complete_ut.cpp | 46 ++++++++++++++++------ 7 files changed, 105 insertions(+), 38 deletions(-) diff --git a/yql/essentials/cfg/tests/gateways-experimental.conf b/yql/essentials/cfg/tests/gateways-experimental.conf index e8adb01227c..fc9754726f7 100644 --- a/yql/essentials/cfg/tests/gateways-experimental.conf +++ b/yql/essentials/cfg/tests/gateways-experimental.conf @@ -99,7 +99,8 @@ YqlCore { "MemberNthOverFlatMap", "ExtractMembersSplitOnOptional", "FilterNullMembersOverJust", - "UnorderedOverSortImproved" + "UnorderedOverSortImproved", + "ExtractCommonPredicatesFromLogicalOps" ] } } diff --git a/yql/essentials/sql/v1/complete/name/cache/byte_size.h b/yql/essentials/sql/v1/complete/name/cache/byte_size.h index f838d364750..feb3e8e7bf9 100644 --- a/yql/essentials/sql/v1/complete/name/cache/byte_size.h +++ b/yql/essentials/sql/v1/complete/name/cache/byte_size.h @@ -5,6 +5,7 @@ #include #include +#include namespace NSQLComplete { @@ -38,6 +39,14 @@ namespace NSQLComplete { } }; + template + struct TByteSize> { + size_t operator()(const TMaybe& x) const noexcept { + return x.Transform([](const T& x) { return TByteSize()(x) - sizeof(T); }).GetOrElse(0) + + sizeof(TMaybe); + } + }; + template concept CByteSized = requires(const T& x) { { TByteSize()(x) } -> std::convertible_to; diff --git a/yql/essentials/sql/v1/complete/name/object/simple/cached/key.h b/yql/essentials/sql/v1/complete/name/object/simple/cached/key.h index c5e678c8d13..79c6736042d 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/cached/key.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/cached/key.h @@ -8,23 +8,23 @@ namespace NSQLComplete { - struct TSchemaListCacheKey { + struct TSchemaDescribeCacheKey { TString Zone; TString Cluster; - TString Folder; + TString Path; friend bool operator==( - const TSchemaListCacheKey& lhs, - const TSchemaListCacheKey& rhs) = default; + const TSchemaDescribeCacheKey& lhs, + const TSchemaDescribeCacheKey& rhs) = default; }; template <> - struct TByteSize { - size_t operator()(const TSchemaListCacheKey& x) const noexcept { + struct TByteSize { + size_t operator()(const TSchemaDescribeCacheKey& x) const noexcept { return sizeof(x) + TByteSize()(x.Zone) + TByteSize()(x.Cluster) + - TByteSize()(x.Folder); + TByteSize()(x.Path); } }; @@ -37,12 +37,19 @@ namespace NSQLComplete { } }; + template <> + struct TByteSize { + size_t operator()(const TTableDetails& x) const noexcept { + return TByteSize>()(x.Columns); + } + }; + } // namespace NSQLComplete template <> -struct THash { - inline size_t operator()(const NSQLComplete::TSchemaListCacheKey& key) const { +struct THash { + inline size_t operator()(const NSQLComplete::TSchemaDescribeCacheKey& key) const { return THash>()( - std::tie(key.Zone, key.Cluster, key.Folder)); + std::tie(key.Zone, key.Cluster, key.Path)); } }; diff --git a/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.cpp b/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.cpp index 00c4515da58..77abd97c576 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.cpp @@ -9,13 +9,16 @@ namespace NSQLComplete { class TSimpleSchema: public ISimpleSchema { public: TSimpleSchema( - ISchemaListCache::TPtr Cache, + TSchemaCaches caches, TString Zone, ISimpleSchema::TPtr Origin) : Zone_(std::move(Zone)) , Origin_(std::move(Origin)) - , Query_(std::move(Cache), [origin = Origin_](const TSchemaListCacheKey& key) { - return origin->List(key.Cluster, key.Folder); + , QueryList_(std::move(caches.List), [origin = Origin_](const TSchemaDescribeCacheKey& key) { + return origin->List(key.Cluster, key.Path); + }) + , QueryDescribeTable_(std::move(caches.DescribeTable), [origin = Origin_](const TSchemaDescribeCacheKey& key) { + return origin->DescribeTable(key.Cluster, key.Path); }) { } @@ -26,24 +29,38 @@ namespace NSQLComplete { NThreading::TFuture> List(TString cluster, TString folder) const override { - return Query_({ + return QueryList_({ .Zone = Zone_, .Cluster = std::move(cluster), - .Folder = std::move(folder), + .Path = std::move(folder), + }); + } + + NThreading::TFuture> + DescribeTable(const TString& cluster, const TString& path) const override { + return QueryDescribeTable_({ + .Zone = Zone_, + .Cluster = cluster, + .Path = path, }); } private: TString Zone_; ISimpleSchema::TPtr Origin_; - TCachedQuery> Query_; + TCachedQuery> QueryList_; + TCachedQuery> QueryDescribeTable_; }; } // namespace ISimpleSchema::TPtr MakeCachedSimpleSchema( ISchemaListCache::TPtr cache, TString zone, ISimpleSchema::TPtr origin) { - return new TSimpleSchema(std::move(cache), std::move(zone), std::move(origin)); + return new TSimpleSchema({.List = std::move(cache)}, std::move(zone), std::move(origin)); + } + + ISimpleSchema::TPtr MakeCachedSimpleSchema(TSchemaCaches caches, TString zone, ISimpleSchema::TPtr origin) { + return new TSimpleSchema(std::move(caches), std::move(zone), std::move(origin)); } } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.h b/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.h index 3c0545fd6bb..4ad428f3cf4 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/cached/schema.h @@ -8,9 +8,19 @@ namespace NSQLComplete { - using ISchemaListCache = ICache>; + using ISchemaListCache = ICache>; + using ISchemaDescribeTableCache = ICache>; + struct TSchemaCaches { + ISchemaListCache::TPtr List; + ISchemaDescribeTableCache::TPtr DescribeTable; + }; + + // TODO(YQL-19747): deprecated, migrate YDB CLI ISimpleSchema::TPtr MakeCachedSimpleSchema( ISchemaListCache::TPtr cache, TString zone, ISimpleSchema::TPtr origin); + ISimpleSchema::TPtr MakeCachedSimpleSchema( + TSchemaCaches caches, TString zone, ISimpleSchema::TPtr origin); + } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp index 576de4dfdc5..d413ee859f0 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp @@ -41,16 +41,12 @@ namespace NSQLComplete { } NThreading::TFuture> List(TString cluster, TString folder) const override { - if (!folder.StartsWith('/')) { - folder.prepend('/'); - } - TVector entries; const THashMap>* tables = nullptr; const TVector* items = nullptr; if ((tables = Folders_.FindPtr(cluster)) && - (items = tables->FindPtr(folder))) { + (items = tables->FindPtr(Qualified(folder)))) { entries = *items; } @@ -64,7 +60,7 @@ namespace NSQLComplete { return NThreading::MakeFuture>(Nothing()); } - auto* details = tables->FindPtr(path); + auto* details = tables->FindPtr(Qualified(path)); if (details == nullptr) { return NThreading::MakeFuture>(Nothing()); } @@ -73,6 +69,13 @@ namespace NSQLComplete { } private: + TString Qualified(TString path) const { + if (!path.StartsWith('/')) { + path.prepend('/'); + } + return path; + } + THashMap>> Folders_; THashMap> Tables_; }; diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp index 90c2d8faeba..6d6d91d0dbb 100644 --- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -1377,40 +1377,60 @@ JOIN yt:$cluster_name.test; Y_UNIT_TEST(CachedSchema) { TLexerSupplier lexer = MakePureLexerSupplier(); - auto cache = MakeLocalCache< - TSchemaListCacheKey, TVector>( - NMonotonic::CreateDefaultMonotonicTimeProvider(), {}); + auto time = NMonotonic::CreateDefaultMonotonicTimeProvider(); + TSchemaCaches caches = { + .List = MakeLocalCache< + TSchemaDescribeCacheKey, TVector>(time, {}), + .DescribeTable = MakeLocalCache< + TSchemaDescribeCacheKey, TMaybe>(time, {}), + }; auto aliceService = MakeSchemaNameService( MakeSimpleSchema( MakeCachedSimpleSchema( - cache, "alice", + caches, "alice", MakeStaticSimpleSchema(TSchemaData{ .Folders = {{"", {{"/", {{"Table", "alice"}}}}}}, + .Tables = {{"", {{"/alice", {{"alice"}}}}}}, })))); auto petyaService = MakeSchemaNameService( MakeSimpleSchema( MakeCachedSimpleSchema( - cache, "petya", + caches, "petya", MakeStaticSimpleSchema(TSchemaData{ .Folders = {{"", {{"/", {{"Table", "petya"}}}}}}, + .Tables = {{"", {{"/petya", {{"petya"}}}}}}, })))); auto aliceEngine = MakeSqlCompletionEngine(lexer, std::move(aliceService)); auto petyaEngine = MakeSqlCompletionEngine(lexer, std::move(petyaService)); TVector empty; - TVector aliceExpected = {{TableName, "`alice`"}}; - TVector petyaExpected = {{TableName, "`petya`"}}; + { + TVector aliceExpected = {{TableName, "`alice`"}}; + TVector petyaExpected = {{TableName, "`petya`"}}; - // Cache is empty - UNIT_ASSERT_VALUES_EQUAL(Complete(aliceEngine, "SELECT * FROM "), empty); - UNIT_ASSERT_VALUES_EQUAL(Complete(petyaEngine, "SELECT * FROM "), empty); + // Cache is empty + UNIT_ASSERT_VALUES_EQUAL(Complete(aliceEngine, "SELECT * FROM "), empty); + UNIT_ASSERT_VALUES_EQUAL(Complete(petyaEngine, "SELECT * FROM "), empty); - // Updates in backround - UNIT_ASSERT_VALUES_EQUAL(Complete(aliceEngine, "SELECT * FROM "), aliceExpected); - UNIT_ASSERT_VALUES_EQUAL(Complete(petyaEngine, "SELECT * FROM "), petyaExpected); + // Updates in backround + UNIT_ASSERT_VALUES_EQUAL(Complete(aliceEngine, "SELECT * FROM "), aliceExpected); + UNIT_ASSERT_VALUES_EQUAL(Complete(petyaEngine, "SELECT * FROM "), petyaExpected); + } + { + TVector aliceExpected = {{ColumnName, "alice"}}; + TVector petyaExpected = {{ColumnName, "petya"}}; + + // Cache is empty + UNIT_ASSERT_VALUES_EQUAL(Complete(aliceEngine, "SELECT a# FROM alice"), empty); + UNIT_ASSERT_VALUES_EQUAL(Complete(petyaEngine, "SELECT p# FROM petya"), empty); + + // Updates in backround + UNIT_ASSERT_VALUES_EQUAL(Complete(aliceEngine, "SELECT a# FROM alice"), aliceExpected); + UNIT_ASSERT_VALUES_EQUAL(Complete(petyaEngine, "SELECT p# FROM petya"), petyaExpected); + } } } // Y_UNIT_TEST_SUITE(SqlCompleteTests) -- cgit v1.3